Skip to content

Commit 96e194f

Browse files
lukeyeagerlukeyeager
committed
Overhaul TravisCI
* Run on Ubuntu 14.04 * Test cuDNN builds * Build with OpenBLAS NOTE: Python3 build only works with CMake
1 parent 10f99e8 commit 96e194f

13 files changed

+292
-219
lines changed

‎.travis.yml

+34-27
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,50 @@
1-
# Use a build matrix to do two builds in parallel:
2-
# one using CMake, and one using make.
1+
dist: trusty
2+
sudo: required
3+
4+
language: cpp
5+
compiler: gcc
6+
37
env:
8+
global:
9+
- NUM_THREADS=4
410
matrix:
5-
- WITH_CUDA=false WITH_CMAKE=false WITH_IO=true
6-
- WITH_CUDA=false WITH_CMAKE=true WITH_IO=true PYTHON_VERSION=3
7-
- WITH_CUDA=true WITH_CMAKE=false WITH_IO=true
8-
- WITH_CUDA=true WITH_CMAKE=true WITH_IO=true
9-
- WITH_CUDA=false WITH_CMAKE=false WITH_IO=false
10-
- WITH_CUDA=false WITH_CMAKE=true WITH_IO=false PYTHON_VERSION=3
11-
# Currently there is no way to install cudnn via apt-get. Uncomment when it's available.
12-
# - WITH_CUDA=true WITH_CMAKE=false WITH_IO=true WITH_CUDNN=true
13-
# - WITH_CUDA=true WITH_CMAKE=true WITH_IO=true WITH_CUDNN=true
11+
# Use a build matrix to test many builds in parallel
12+
# envvar defaults:
13+
# WITH_CMAKE: false
14+
# WITH_PYTHON3: false
15+
# WITH_IO: true
16+
# WITH_CUDA: false
17+
# WITH_CUDNN: false
18+
- BUILD_NAME="default-make"
19+
# - BUILD_NAME="python3-make" WITH_PYTHON3=true
20+
- BUILD_NAME="no-io-make" WITH_IO=false
21+
- BUILD_NAME="cuda-make" WITH_CUDA=true
22+
- BUILD_NAME="cudnn-make" WITH_CUDA=true WITH_CUDNN=true
1423

15-
language: cpp
24+
- BUILD_NAME="default-cmake" WITH_CMAKE=true
25+
- BUILD_NAME="python3-cmake" WITH_CMAKE=true WITH_PYTHON3=true
26+
- BUILD_NAME="no-io-cmake" WITH_CMAKE=true WITH_IO=false
27+
- BUILD_NAME="cuda-cmake" WITH_CMAKE=true WITH_CUDA=true
28+
- BUILD_NAME="cudnn-cmake" WITH_CMAKE=true WITH_CUDA=true WITH_CUDNN=true
1629

17-
# Cache Ubuntu apt packages.
1830
cache:
1931
apt: true
20-
directories:
21-
- /home/travis/miniconda
22-
- /home/travis/miniconda2
23-
- /home/travis/miniconda3
24-
25-
compiler: gcc
2632

2733
before_install:
28-
- export NUM_THREADS=4
29-
- export SCRIPTS=./scripts/travis
30-
- export CONDA_DIR="/home/travis/miniconda$PYTHON_VERSION"
34+
- source ./scripts/travis/defaults.sh
3135

3236
install:
33-
- sudo -E $SCRIPTS/travis_install.sh
37+
- sudo -E ./scripts/travis/install-deps.sh
38+
- ./scripts/travis/setup-venv.sh ~/venv
39+
- source ~/venv/bin/activate
40+
- ./scripts/travis/install-python-deps.sh
3441

3542
before_script:
36-
- export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64:$CONDA_DIR/lib
37-
- export PATH=$CONDA_DIR/bin:$PATH
38-
- if ! $WITH_CMAKE; then $SCRIPTS/travis_setup_makefile_config.sh; fi
43+
- ./scripts/travis/configure.sh
3944

40-
script: $SCRIPTS/travis_build_and_test.sh
45+
script:
46+
- ./scripts/travis/build.sh
47+
- ./scripts/travis/test.sh
4148

4249
notifications:
4350
# Emails are sent to the committer's git-configured email address by default,

‎scripts/travis/build.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
# build the project
3+
4+
BASEDIR=$(dirname $0)
5+
source $BASEDIR/defaults.sh
6+
7+
if ! $WITH_CMAKE ; then
8+
make --jobs $NUM_THREADS all test pycaffe warn
9+
else
10+
cd build
11+
make --jobs $NUM_THREADS all test.testbin
12+
fi
13+
make lint

‎scripts/travis/configure-cmake.sh

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# CMake configuration
2+
3+
mkdir -p build
4+
cd build
5+
6+
ARGS="-DCMAKE_BUILD_TYPE=Release -DBLAS=Open"
7+
8+
if $WITH_PYTHON3 ; then
9+
ARGS="$ARGS -Dpython_version=3"
10+
fi
11+
12+
if $WITH_IO ; then
13+
ARGS="$ARGS -DUSE_OPENCV=On -DUSE_LMDB=On -DUSE_LEVELDB=On"
14+
else
15+
ARGS="$ARGS -DUSE_OPENCV=Off -DUSE_LMDB=Off -DUSE_LEVELDB=Off"
16+
fi
17+
18+
if $WITH_CUDA ; then
19+
# Only build SM50
20+
ARGS="$ARGS -DCPU_ONLY=Off -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=\"50\" -DCUDA_ARCH_PTX=\"\""
21+
else
22+
ARGS="$ARGS -DCPU_ONLY=On"
23+
fi
24+
25+
if $WITH_CUDNN ; then
26+
ARGS="$ARGS -DUSE_CUDNN=On"
27+
else
28+
ARGS="$ARGS -DUSE_CUDNN=Off"
29+
fi
30+
31+
cmake .. $ARGS
32+

‎scripts/travis/configure-make.sh

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# raw Makefile configuration
2+
3+
LINE () {
4+
echo "$@" >> Makefile.config
5+
}
6+
7+
cp Makefile.config.example Makefile.config
8+
9+
LINE "BLAS := open"
10+
LINE "WITH_PYTHON_LAYER := 1"
11+
12+
if $WITH_PYTHON3 ; then
13+
# TODO(lukeyeager) this path is currently disabled because of test errors like:
14+
# ImportError: dynamic module does not define init function (PyInit__caffe)
15+
LINE "PYTHON_LIBRARIES := python3.4m boost_python-py34"
16+
LINE "PYTHON_INCLUDE := /usr/include/python3.4 /usr/lib/python3/dist-packages/numpy/core/include"
17+
LINE "INCLUDE_DIRS := \$(INCLUDE_DIRS) \$(PYTHON_INCLUDE)"
18+
fi
19+
20+
if ! $WITH_IO ; then
21+
LINE "USE_OPENCV := 0"
22+
LINE "USE_LEVELDB := 0"
23+
LINE "USE_LMDB := 0"
24+
fi
25+
26+
if $WITH_CUDA ; then
27+
# Only build SM50
28+
LINE "CUDA_ARCH := -gencode arch=compute_50,code=sm_50"
29+
else
30+
LINE "CPU_ONLY := 1"
31+
fi
32+
33+
if $WITH_CUDNN ; then
34+
LINE "USE_CUDNN := 1"
35+
fi
36+

‎scripts/travis/configure.sh

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
# configure the project
3+
4+
BASEDIR=$(dirname $0)
5+
source $BASEDIR/defaults.sh
6+
7+
if ! $WITH_CMAKE ; then
8+
source $BASEDIR/configure-make.sh
9+
else
10+
source $BASEDIR/configure-cmake.sh
11+
fi

‎scripts/travis/defaults.sh

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
# set default environment variables
3+
4+
set -e
5+
6+
WITH_CMAKE=${WITH_CMAKE:-false}
7+
WITH_PYTHON3=${WITH_PYTHON3:-false}
8+
WITH_IO=${WITH_IO:-true}
9+
WITH_CUDA=${WITH_CUDA:-false}
10+
WITH_CUDNN=${WITH_CUDNN:-false}

‎scripts/travis/install-deps.sh

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
# install dependencies
3+
# (this script must be run as root)
4+
5+
BASEDIR=$(dirname $0)
6+
source $BASEDIR/defaults.sh
7+
8+
apt-get -y update
9+
apt-get install -y --no-install-recommends \
10+
build-essential \
11+
libboost-filesystem-dev \
12+
libboost-python-dev \
13+
libboost-system-dev \
14+
libboost-thread-dev \
15+
libgflags-dev \
16+
libgoogle-glog-dev \
17+
libhdf5-serial-dev \
18+
libopenblas-dev \
19+
python-virtualenv \
20+
wget
21+
22+
if $WITH_CMAKE ; then
23+
apt-get install -y --no-install-recommends cmake
24+
fi
25+
26+
if ! $WITH_PYTHON3 ; then
27+
# Python2
28+
apt-get install -y --no-install-recommends \
29+
libprotobuf-dev \
30+
protobuf-compiler \
31+
python-dev \
32+
python-numpy \
33+
python-protobuf \
34+
python-skimage
35+
else
36+
# Python3
37+
apt-get install -y --no-install-recommends \
38+
python3-dev \
39+
python3-numpy \
40+
python3-skimage
41+
42+
# build Protobuf3 since it's needed for Python3
43+
echo "Building protobuf3 from source ..."
44+
pushd .
45+
PROTOBUF3_DIR=~/protobuf3-build
46+
rm -rf $PROTOBUF3_DIR
47+
mkdir $PROTOBUF3_DIR
48+
49+
# install some more dependencies required to build protobuf3
50+
apt-get install -y --no-install-recommends \
51+
curl \
52+
dh-autoreconf \
53+
unzip
54+
55+
wget https://github.com/google/protobuf/archive/v3.0.0-beta-3.tar.gz -O protobuf3.tar.gz
56+
tar -xzf protobuf3.tar.gz -C $PROTOBUF3_DIR --strip 1
57+
rm protobuf3.tar.gz
58+
cd $PROTOBUF3_DIR
59+
./autogen.sh
60+
./configure --prefix=/usr
61+
make --jobs=$NUM_THREADS
62+
make install
63+
popd
64+
fi
65+
66+
if $WITH_IO ; then
67+
apt-get install -y --no-install-recommends \
68+
libleveldb-dev \
69+
liblmdb-dev \
70+
libopencv-dev \
71+
libsnappy-dev
72+
fi
73+
74+
if $WITH_CUDA ; then
75+
# install repo packages
76+
CUDA_REPO_PKG=cuda-repo-ubuntu1404_7.5-18_amd64.deb
77+
wget http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1404/x86_64/$CUDA_REPO_PKG
78+
dpkg -i $CUDA_REPO_PKG
79+
rm $CUDA_REPO_PKG
80+
81+
if $WITH_CUDNN ; then
82+
ML_REPO_PKG=nvidia-machine-learning-repo_4.0-2_amd64.deb
83+
wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1404/x86_64/$ML_REPO_PKG
84+
dpkg -i $ML_REPO_PKG
85+
fi
86+
87+
# update package lists
88+
apt-get -y update
89+
90+
# install packages
91+
CUDA_PKG_VERSION="7-5"
92+
CUDA_VERSION="7.5"
93+
apt-get install -y --no-install-recommends \
94+
cuda-core-$CUDA_PKG_VERSION \
95+
cuda-cudart-dev-$CUDA_PKG_VERSION \
96+
cuda-cublas-dev-$CUDA_PKG_VERSION \
97+
cuda-curand-dev-$CUDA_PKG_VERSION
98+
# manually create CUDA symlink
99+
ln -s /usr/local/cuda-$CUDA_VERSION /usr/local/cuda
100+
101+
if $WITH_CUDNN ; then
102+
apt-get install -y --no-install-recommends libcudnn5-dev
103+
fi
104+
fi
105+

‎scripts/travis/install-python-deps.sh

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
# install extra Python dependencies
3+
# (must come after setup-venv)
4+
5+
BASEDIR=$(dirname $0)
6+
source $BASEDIR/defaults.sh
7+
8+
if ! $WITH_PYTHON3 ; then
9+
# Python2
10+
:
11+
else
12+
# Python3
13+
pip install --pre protobuf==3.0.0b3
14+
fi

‎scripts/travis/setup-venv.sh

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
# setup a Python virtualenv
3+
# (must come after install-deps)
4+
5+
BASEDIR=$(dirname $0)
6+
source $BASEDIR/defaults.sh
7+
8+
VENV_DIR=${1:-~/venv}
9+
10+
# setup our own virtualenv
11+
if $WITH_PYTHON3; then
12+
PYTHON_EXE='/usr/bin/python3'
13+
else
14+
PYTHON_EXE='/usr/bin/python2'
15+
fi
16+
17+
# use --system-site-packages so that Python will use deb packages
18+
virtualenv $VENV_DIR -p $PYTHON_EXE --system-site-packages

‎scripts/travis/test.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
# test the project
3+
4+
BASEDIR=$(dirname $0)
5+
source $BASEDIR/defaults.sh
6+
7+
if $WITH_CUDA ; then
8+
echo "Skipping tests for CUDA build"
9+
exit 0
10+
fi
11+
12+
if ! $WITH_CMAKE ; then
13+
make runtest
14+
make pytest
15+
else
16+
cd build
17+
make runtest
18+
make pytest
19+
fi

0 commit comments

Comments
 (0)