From 1db8592d2d98ae223730fe7fe3b0571611f8b9c8 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 15:58:17 -0500 Subject: [PATCH 01/13] For Python3, just check if an instance of str --- nbflow/extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbflow/extractor.py b/nbflow/extractor.py index 6341382..6a20d7c 100644 --- a/nbflow/extractor.py +++ b/nbflow/extractor.py @@ -57,7 +57,7 @@ def get_dependencies(self, dirnames): sources = [self.resolve_path(filename, x) for x in params['__depends__']] targets = params['__dest__'] - if not hasattr(targets, '__iter__'): + if isinstance(targets, str): if targets is None: targets = [] else: From 63db9a432dc687c97c8d4406c05837a539590ec7 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 15:59:03 -0500 Subject: [PATCH 02/13] Add doit to requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ee81e4d..56314c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ traitlets nbconvert==4.1 scons -ipykernel \ No newline at end of file +ipykernel +doit>=0.29 From 8dec722f366fee6bee8b8201c02c99544d81bf97 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 16:02:26 -0500 Subject: [PATCH 03/13] Rename example to scons-example --- nbflow/{example => scons-example}/README.md | 0 nbflow/{example => scons-example}/SConstruct | 0 nbflow/{example => scons-example}/analyses/analyze_data.ipynb | 0 nbflow/{example => scons-example}/analyses/gen_data.ipynb | 0 nbflow/{example => scons-example}/results/data.json | 0 nbflow/{example => scons-example}/results/stats.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename nbflow/{example => scons-example}/README.md (100%) rename nbflow/{example => scons-example}/SConstruct (100%) rename nbflow/{example => scons-example}/analyses/analyze_data.ipynb (100%) rename nbflow/{example => scons-example}/analyses/gen_data.ipynb (100%) rename nbflow/{example => scons-example}/results/data.json (100%) rename nbflow/{example => scons-example}/results/stats.json (100%) diff --git a/nbflow/example/README.md b/nbflow/scons-example/README.md similarity index 100% rename from nbflow/example/README.md rename to nbflow/scons-example/README.md diff --git a/nbflow/example/SConstruct b/nbflow/scons-example/SConstruct similarity index 100% rename from nbflow/example/SConstruct rename to nbflow/scons-example/SConstruct diff --git a/nbflow/example/analyses/analyze_data.ipynb b/nbflow/scons-example/analyses/analyze_data.ipynb similarity index 100% rename from nbflow/example/analyses/analyze_data.ipynb rename to nbflow/scons-example/analyses/analyze_data.ipynb diff --git a/nbflow/example/analyses/gen_data.ipynb b/nbflow/scons-example/analyses/gen_data.ipynb similarity index 100% rename from nbflow/example/analyses/gen_data.ipynb rename to nbflow/scons-example/analyses/gen_data.ipynb diff --git a/nbflow/example/results/data.json b/nbflow/scons-example/results/data.json similarity index 100% rename from nbflow/example/results/data.json rename to nbflow/scons-example/results/data.json diff --git a/nbflow/example/results/stats.json b/nbflow/scons-example/results/stats.json similarity index 100% rename from nbflow/example/results/stats.json rename to nbflow/scons-example/results/stats.json From 3ee7e5b955c0f61f337931c41cfa825fafde3020 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 16:02:52 -0500 Subject: [PATCH 04/13] First run of task builders for pydoit --- nbflow/doit.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 nbflow/doit.py diff --git a/nbflow/doit.py b/nbflow/doit.py new file mode 100644 index 0000000..e6bbad2 --- /dev/null +++ b/nbflow/doit.py @@ -0,0 +1,58 @@ +import json +import subprocess as sp +import sys + +from doit.action import CmdAction +from doit.task import clean_targets + +from .extractor import DependencyExtractor +from ._version import __version__ + + +def build_notebook_cmd(notebook): + cmd = [ + "jupyter", "nbconvert", + "--log-level=ERROR", + "--ExecutePreprocessor.timeout=120", + "--execute", + "--inplace", + "--to", "notebook", + #"--output", notebook, + notebook + ] + return ' '.join(cmd) + + +def touch_cmds(target): + for t in target: + yield 'touch {0}'.format(str(t)) + + +def get_cmds(source, target): + yield build_notebook_cmd(source) + yield from touch_cmds(target) + + +def create_build_tasks(directories): + parsed = sp.check_output([sys.executable, "-m", "nbflow"] + directories, + universal_newlines=True) + DEPENDENCIES = json.loads(parsed) + for script in DEPENDENCIES: + deps = DEPENDENCIES[script] + sources = deps['sources'] + if len(deps['targets']) == 0: + #targets = ['.phony_{}'.format(script)] + targets = [] + else: + targets = deps['targets'] + + name = 'build-notebook:{0}'.format(script) + file_dep = [script] + sources + actions = [CmdAction(cmd) for cmd in get_cmds(script, targets)] + + yield {'name': name, + 'actions': actions, + 'targets': targets, + 'file_dep': file_dep, + 'clean': [clean_targets]} + From 21970f1791d0fc3236bd3fe64c178aa183cff698 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 16:05:28 -0500 Subject: [PATCH 05/13] Add a pydoit example --- nbflow/doit-example/README.md | 7 + .../doit-example/analyses/analyze_data.ipynb | 128 ++++++++++++++++++ nbflow/doit-example/analyses/gen_data.ipynb | 98 ++++++++++++++ nbflow/doit-example/dodo.py | 5 + nbflow/doit-example/results/data.json | 1 + nbflow/doit-example/results/stats.json | 1 + 6 files changed, 240 insertions(+) create mode 100644 nbflow/doit-example/README.md create mode 100644 nbflow/doit-example/analyses/analyze_data.ipynb create mode 100644 nbflow/doit-example/analyses/gen_data.ipynb create mode 100644 nbflow/doit-example/dodo.py create mode 100644 nbflow/doit-example/results/data.json create mode 100644 nbflow/doit-example/results/stats.json diff --git a/nbflow/doit-example/README.md b/nbflow/doit-example/README.md new file mode 100644 index 0000000..7d63b48 --- /dev/null +++ b/nbflow/doit-example/README.md @@ -0,0 +1,7 @@ +# Example nbflow setup + +To execute the notebooks in this example, run from this directory: + +``` +doit +``` diff --git a/nbflow/doit-example/analyses/analyze_data.ipynb b/nbflow/doit-example/analyses/analyze_data.ipynb new file mode 100644 index 0000000..4ed6354 --- /dev/null +++ b/nbflow/doit-example/analyses/analyze_data.ipynb @@ -0,0 +1,128 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "__depends__ = [\"../results/data.json\"]\n", + "__dest__ = \"../results/stats.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open(\"../results/data.json\", \"r\") as fh:\n", + " data = json.load(fh)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.7947434550704049,\n", + " 0.7689497615073635,\n", + " 0.4055562370238408,\n", + " 0.1437725456662664,\n", + " 0.06290778106817208,\n", + " 0.9690115597726553,\n", + " 0.512838996781445,\n", + " 0.09813919031940244,\n", + " 0.31852186939224447,\n", + " 0.41326319707820003]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'mean': 0.49688692836011145, 'var': 0.08984597204318795}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mean = sum(data) / len(data)\n", + "var = sum(((x - mean) ** 2) / (len(data) - 1) for x in data)\n", + "results = dict(mean=mean, var=var)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "with open(__dest__, \"w\") as fh:\n", + " json.dump(results, fh)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/nbflow/doit-example/analyses/gen_data.ipynb b/nbflow/doit-example/analyses/gen_data.ipynb new file mode 100644 index 0000000..64f48a7 --- /dev/null +++ b/nbflow/doit-example/analyses/gen_data.ipynb @@ -0,0 +1,98 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "__depends__ = []\n", + "__dest__ = \"../results/data.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "import random\n", + "import json\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.7947434550704049,\n", + " 0.7689497615073635,\n", + " 0.4055562370238408,\n", + " 0.1437725456662664,\n", + " 0.06290778106817208,\n", + " 0.9690115597726553,\n", + " 0.512838996781445,\n", + " 0.09813919031940244,\n", + " 0.31852186939224447,\n", + " 0.41326319707820003]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "random.seed(\"92831\")\n", + "data = [random.random() for x in range(100)]\n", + "data[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "if not os.path.exists(\"../results\"):\n", + " os.mkdir(\"../results\")\n", + "\n", + "with open(__dest__, \"w\") as fh:\n", + " json.dump(data, fh)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/nbflow/doit-example/dodo.py b/nbflow/doit-example/dodo.py new file mode 100644 index 0000000..81b9803 --- /dev/null +++ b/nbflow/doit-example/dodo.py @@ -0,0 +1,5 @@ +import os +from nbflow.doit import create_build_tasks + +def task_nbflow(): + yield from create_build_tasks(['analyses']) diff --git a/nbflow/doit-example/results/data.json b/nbflow/doit-example/results/data.json new file mode 100644 index 0000000..04f727b --- /dev/null +++ b/nbflow/doit-example/results/data.json @@ -0,0 +1 @@ +[0.7947434550704049, 0.7689497615073635, 0.4055562370238408, 0.1437725456662664, 0.06290778106817208, 0.9690115597726553, 0.512838996781445, 0.09813919031940244, 0.31852186939224447, 0.41326319707820003, 0.385612433500892, 0.24209750775116456, 0.29082206251730214, 0.30124406210666455, 0.8240710383905946, 0.10523477581705942, 0.23327614000219432, 0.6797287458288424, 0.5510274057377629, 0.24377157615518152, 0.42657124031674587, 0.5162762904164121, 0.6854427487166, 0.6977893969788851, 0.35588141087851544, 0.18132047504217308, 0.18000658496189414, 0.8440723134093764, 0.3043735959401148, 0.0496267641631889, 0.5261456814060039, 0.10523700829988625, 0.35204390444062006, 0.343477373394917, 0.1231406560434184, 0.7121803271620056, 0.4485702182752801, 0.8968689393414282, 0.8601693399743452, 0.9564706610947532, 0.8724397884450427, 0.6700216727677132, 0.5257850379386515, 0.13559082556225643, 0.7072264774014883, 0.22803465958913371, 0.9657833400104027, 0.5848204795795097, 0.02451351622485054, 0.8433809250559492, 0.11028115721837406, 0.6889894142668991, 0.7840364708405744, 0.6926017709187364, 0.30056326600575467, 0.20549338035132092, 0.3913191042744558, 0.06588572432899564, 0.9558437556319708, 0.5415757382513219, 0.06978291698630412, 0.6373331189932064, 0.022281441929325907, 0.4110883299332122, 0.5573646890623988, 0.8478386511492215, 0.044382721435622274, 0.5844098508897433, 0.013114313589633442, 0.6089826842992508, 0.30717566153718523, 0.8170659104579339, 0.17566832286738, 0.2604875020899934, 0.9162989972841736, 0.22290419127691607, 0.9774836151565892, 0.24911332819022292, 0.7816282468374476, 0.12086278849786802, 0.932322453430037, 0.3853365385491174, 0.7704778800639159, 0.529923891586678, 0.9464024806024787, 0.32206152193330584, 0.8658112284584186, 0.7980987362057588, 0.6613687795953515, 0.69054877216148, 0.8613510596942656, 0.7506497320539183, 0.8935225381758013, 0.9790981646563037, 0.39818066454463785, 0.1886620300227192, 0.8132203301342672, 0.020239491579228552, 0.7663120626263462, 0.2853954210658719] \ No newline at end of file diff --git a/nbflow/doit-example/results/stats.json b/nbflow/doit-example/results/stats.json new file mode 100644 index 0000000..01df384 --- /dev/null +++ b/nbflow/doit-example/results/stats.json @@ -0,0 +1 @@ +{"mean": 0.49688692836011145, "var": 0.08984597204318795} \ No newline at end of file From 7d51192704e4f372d43a6d1cdb9718bf9fd28262 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 16:05:42 -0500 Subject: [PATCH 06/13] Update gitignore --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 4f0aa02..bd3a7de 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,10 @@ target/ # Scons .sconsign.dblite + +# doit +.doit.db +.doit.db.db + +# vim +*.swp From d5f48c8efd7cf4fe7e60c0c1594428f873efeba3 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Fri, 15 Jul 2016 16:22:55 -0500 Subject: [PATCH 07/13] Update scons example location --- nbflow/tests/test_nbflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbflow/tests/test_nbflow.py b/nbflow/tests/test_nbflow.py index 4a48f3f..a09087c 100644 --- a/nbflow/tests/test_nbflow.py +++ b/nbflow/tests/test_nbflow.py @@ -12,7 +12,7 @@ def test_nbflow_no_args(temp_cwd): def test_example(temp_cwd): # copy example files - root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "example")) + root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "scons-example")) shutil.copytree(os.path.join(root, "analyses"), "analyses") shutil.copy(os.path.join(root, "SConstruct"), "SConstruct") clear_notebooks("analyses") From c0cbf013e2cb4af098607409b5b6729bd4cd164a Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 09:55:38 -0700 Subject: [PATCH 08/13] Make extractor py23 compat - fix improper None target handling --- nbflow/extractor.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nbflow/extractor.py b/nbflow/extractor.py index 6a20d7c..b928f17 100644 --- a/nbflow/extractor.py +++ b/nbflow/extractor.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import os import glob import json @@ -57,11 +59,10 @@ def get_dependencies(self, dirnames): sources = [self.resolve_path(filename, x) for x in params['__depends__']] targets = params['__dest__'] - if isinstance(targets, str): - if targets is None: - targets = [] - else: - targets = [targets] + if targets is None: + targets = [] + elif isinstance(targets, str): + targets = [targets] targets = [self.resolve_path(filename, x) for x in targets] dependencies[os.path.join(dirname, '{}.ipynb'.format(modname))] = { From 5f34b34ae04056f89872dc4f6ae589fe2321d077 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 09:56:37 -0700 Subject: [PATCH 09/13] Remove imports from doit package; remove yield froms for py23 compat --- nbflow/doit.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nbflow/doit.py b/nbflow/doit.py index e6bbad2..3174d10 100644 --- a/nbflow/doit.py +++ b/nbflow/doit.py @@ -1,10 +1,9 @@ +from __future__ import print_function + import json import subprocess as sp import sys -from doit.action import CmdAction -from doit.task import clean_targets - from .extractor import DependencyExtractor from ._version import __version__ @@ -28,12 +27,13 @@ def touch_cmds(target): yield 'touch {0}'.format(str(t)) -def get_cmds(source, target): - yield build_notebook_cmd(source) - yield from touch_cmds(target) +def get_cmds(notebook, target): + yield build_notebook_cmd(notebook) + for cmd in touch_cmds(target): + yield cmd -def create_build_tasks(directories): +def setup(directories): parsed = sp.check_output([sys.executable, "-m", "nbflow"] + directories, universal_newlines=True) DEPENDENCIES = json.loads(parsed) @@ -48,11 +48,11 @@ def create_build_tasks(directories): name = 'build-notebook:{0}'.format(script) file_dep = [script] + sources - actions = [CmdAction(cmd) for cmd in get_cmds(script, targets)] - + actions = list(get_cmds(script, targets)) + yield {'name': name, 'actions': actions, 'targets': targets, 'file_dep': file_dep, - 'clean': [clean_targets]} + 'clean': True} From d0c7c7129d9f3c1d4265438f8ce57b58c4c35ad6 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 09:56:59 -0700 Subject: [PATCH 10/13] Remove yield froms --- nbflow/doit-example/dodo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nbflow/doit-example/dodo.py b/nbflow/doit-example/dodo.py index 81b9803..723c319 100644 --- a/nbflow/doit-example/dodo.py +++ b/nbflow/doit-example/dodo.py @@ -1,5 +1,6 @@ import os -from nbflow.doit import create_build_tasks +from nbflow import doit def task_nbflow(): - yield from create_build_tasks(['analyses']) + for task in doit.setup(['analyses']): + yield task From d1e972cfefa03cd0f0df248ac006a57e20da1c38 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 09:58:42 -0700 Subject: [PATCH 11/13] Remove doit and scons from requirements --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 56314c3..e711ee4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ traitlets nbconvert==4.1 -scons ipykernel -doit>=0.29 From 762acd5fa48f0fa16ccf86d9e6b6c060a2dcc2a2 Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 10:03:34 -0700 Subject: [PATCH 12/13] Merge example directories --- nbflow/doit-example/README.md | 7 - .../doit-example/analyses/analyze_data.ipynb | 128 ------------------ nbflow/doit-example/analyses/gen_data.ipynb | 98 -------------- nbflow/doit-example/results/stats.json | 1 - nbflow/{scons-example => example}/README.md | 0 nbflow/{scons-example => example}/SConstruct | 0 .../analyses/analyze_data.ipynb | 0 .../analyses/gen_data.ipynb | 0 nbflow/{doit-example => example}/dodo.py | 0 .../results/data.json | 0 .../results/stats.json | 0 nbflow/scons-example/results/data.json | 1 - 12 files changed, 235 deletions(-) delete mode 100644 nbflow/doit-example/README.md delete mode 100644 nbflow/doit-example/analyses/analyze_data.ipynb delete mode 100644 nbflow/doit-example/analyses/gen_data.ipynb delete mode 100644 nbflow/doit-example/results/stats.json rename nbflow/{scons-example => example}/README.md (100%) rename nbflow/{scons-example => example}/SConstruct (100%) rename nbflow/{scons-example => example}/analyses/analyze_data.ipynb (100%) rename nbflow/{scons-example => example}/analyses/gen_data.ipynb (100%) rename nbflow/{doit-example => example}/dodo.py (100%) rename nbflow/{doit-example => example}/results/data.json (100%) rename nbflow/{scons-example => example}/results/stats.json (100%) delete mode 100644 nbflow/scons-example/results/data.json diff --git a/nbflow/doit-example/README.md b/nbflow/doit-example/README.md deleted file mode 100644 index 7d63b48..0000000 --- a/nbflow/doit-example/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Example nbflow setup - -To execute the notebooks in this example, run from this directory: - -``` -doit -``` diff --git a/nbflow/doit-example/analyses/analyze_data.ipynb b/nbflow/doit-example/analyses/analyze_data.ipynb deleted file mode 100644 index 4ed6354..0000000 --- a/nbflow/doit-example/analyses/analyze_data.ipynb +++ /dev/null @@ -1,128 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "__depends__ = [\"../results/data.json\"]\n", - "__dest__ = \"../results/stats.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "with open(\"../results/data.json\", \"r\") as fh:\n", - " data = json.load(fh)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.7947434550704049,\n", - " 0.7689497615073635,\n", - " 0.4055562370238408,\n", - " 0.1437725456662664,\n", - " 0.06290778106817208,\n", - " 0.9690115597726553,\n", - " 0.512838996781445,\n", - " 0.09813919031940244,\n", - " 0.31852186939224447,\n", - " 0.41326319707820003]" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "data[:10]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'mean': 0.49688692836011145, 'var': 0.08984597204318795}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mean = sum(data) / len(data)\n", - "var = sum(((x - mean) ** 2) / (len(data) - 1) for x in data)\n", - "results = dict(mean=mean, var=var)\n", - "results" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "with open(__dest__, \"w\") as fh:\n", - " json.dump(results, fh)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.5.2" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/nbflow/doit-example/analyses/gen_data.ipynb b/nbflow/doit-example/analyses/gen_data.ipynb deleted file mode 100644 index 64f48a7..0000000 --- a/nbflow/doit-example/analyses/gen_data.ipynb +++ /dev/null @@ -1,98 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "__depends__ = []\n", - "__dest__ = \"../results/data.json\"" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "import random\n", - "import json\n", - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[0.7947434550704049,\n", - " 0.7689497615073635,\n", - " 0.4055562370238408,\n", - " 0.1437725456662664,\n", - " 0.06290778106817208,\n", - " 0.9690115597726553,\n", - " 0.512838996781445,\n", - " 0.09813919031940244,\n", - " 0.31852186939224447,\n", - " 0.41326319707820003]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "random.seed(\"92831\")\n", - "data = [random.random() for x in range(100)]\n", - "data[:10]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "if not os.path.exists(\"../results\"):\n", - " os.mkdir(\"../results\")\n", - "\n", - "with open(__dest__, \"w\") as fh:\n", - " json.dump(data, fh)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 2", - "language": "python", - "name": "python2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.11" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/nbflow/doit-example/results/stats.json b/nbflow/doit-example/results/stats.json deleted file mode 100644 index 01df384..0000000 --- a/nbflow/doit-example/results/stats.json +++ /dev/null @@ -1 +0,0 @@ -{"mean": 0.49688692836011145, "var": 0.08984597204318795} \ No newline at end of file diff --git a/nbflow/scons-example/README.md b/nbflow/example/README.md similarity index 100% rename from nbflow/scons-example/README.md rename to nbflow/example/README.md diff --git a/nbflow/scons-example/SConstruct b/nbflow/example/SConstruct similarity index 100% rename from nbflow/scons-example/SConstruct rename to nbflow/example/SConstruct diff --git a/nbflow/scons-example/analyses/analyze_data.ipynb b/nbflow/example/analyses/analyze_data.ipynb similarity index 100% rename from nbflow/scons-example/analyses/analyze_data.ipynb rename to nbflow/example/analyses/analyze_data.ipynb diff --git a/nbflow/scons-example/analyses/gen_data.ipynb b/nbflow/example/analyses/gen_data.ipynb similarity index 100% rename from nbflow/scons-example/analyses/gen_data.ipynb rename to nbflow/example/analyses/gen_data.ipynb diff --git a/nbflow/doit-example/dodo.py b/nbflow/example/dodo.py similarity index 100% rename from nbflow/doit-example/dodo.py rename to nbflow/example/dodo.py diff --git a/nbflow/doit-example/results/data.json b/nbflow/example/results/data.json similarity index 100% rename from nbflow/doit-example/results/data.json rename to nbflow/example/results/data.json diff --git a/nbflow/scons-example/results/stats.json b/nbflow/example/results/stats.json similarity index 100% rename from nbflow/scons-example/results/stats.json rename to nbflow/example/results/stats.json diff --git a/nbflow/scons-example/results/data.json b/nbflow/scons-example/results/data.json deleted file mode 100644 index 04f727b..0000000 --- a/nbflow/scons-example/results/data.json +++ /dev/null @@ -1 +0,0 @@ -[0.7947434550704049, 0.7689497615073635, 0.4055562370238408, 0.1437725456662664, 0.06290778106817208, 0.9690115597726553, 0.512838996781445, 0.09813919031940244, 0.31852186939224447, 0.41326319707820003, 0.385612433500892, 0.24209750775116456, 0.29082206251730214, 0.30124406210666455, 0.8240710383905946, 0.10523477581705942, 0.23327614000219432, 0.6797287458288424, 0.5510274057377629, 0.24377157615518152, 0.42657124031674587, 0.5162762904164121, 0.6854427487166, 0.6977893969788851, 0.35588141087851544, 0.18132047504217308, 0.18000658496189414, 0.8440723134093764, 0.3043735959401148, 0.0496267641631889, 0.5261456814060039, 0.10523700829988625, 0.35204390444062006, 0.343477373394917, 0.1231406560434184, 0.7121803271620056, 0.4485702182752801, 0.8968689393414282, 0.8601693399743452, 0.9564706610947532, 0.8724397884450427, 0.6700216727677132, 0.5257850379386515, 0.13559082556225643, 0.7072264774014883, 0.22803465958913371, 0.9657833400104027, 0.5848204795795097, 0.02451351622485054, 0.8433809250559492, 0.11028115721837406, 0.6889894142668991, 0.7840364708405744, 0.6926017709187364, 0.30056326600575467, 0.20549338035132092, 0.3913191042744558, 0.06588572432899564, 0.9558437556319708, 0.5415757382513219, 0.06978291698630412, 0.6373331189932064, 0.022281441929325907, 0.4110883299332122, 0.5573646890623988, 0.8478386511492215, 0.044382721435622274, 0.5844098508897433, 0.013114313589633442, 0.6089826842992508, 0.30717566153718523, 0.8170659104579339, 0.17566832286738, 0.2604875020899934, 0.9162989972841736, 0.22290419127691607, 0.9774836151565892, 0.24911332819022292, 0.7816282468374476, 0.12086278849786802, 0.932322453430037, 0.3853365385491174, 0.7704778800639159, 0.529923891586678, 0.9464024806024787, 0.32206152193330584, 0.8658112284584186, 0.7980987362057588, 0.6613687795953515, 0.69054877216148, 0.8613510596942656, 0.7506497320539183, 0.8935225381758013, 0.9790981646563037, 0.39818066454463785, 0.1886620300227192, 0.8132203301342672, 0.020239491579228552, 0.7663120626263462, 0.2853954210658719] \ No newline at end of file From 65cf7b8f22332307af6963fbe88f88a0a008733c Mon Sep 17 00:00:00 2001 From: Camille Scott Date: Sat, 16 Jul 2016 10:19:01 -0700 Subject: [PATCH 13/13] Because doit and scons removed from reqs file, add them to install section of travis yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ab74573..f5ff061 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,5 +7,6 @@ before_install: install: - pip install -U pip setuptools - PIP_FIND_LINKS=~/travis-wheels/wheelhouse pip install -r dev-requirements.txt -e . + - pip install scons doit script: - py.test -v -x