diff --git a/LICENSE b/COPYING
similarity index 100%
rename from LICENSE
rename to COPYING
diff --git a/README.md b/README.md
index a9ae3aa0f9..3a57ad6be1 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
- 
+ 
@@ -12,7 +12,7 @@
- 
+
```bash
@@ -22,7 +22,7 @@ pip install pytorch-lightning
## Docs
In progress. Documenting now!
-## Disclaimer
+## Disclaimer
This is a research tool I built for myself internally while doing my PhD. The API is not 100% production quality, but my hope is that by open-sourcing, we can all get it there (I don't have too much time nowadays to write production-level code).
## What is it?
@@ -38,7 +38,7 @@ Your model.
2. Run the validation loop.
3. Run the testing loop.
4. Early stopping.
-5. Learning rate annealing.
+5. Learning rate annealing.
6. Can train complex models like GANs or anything with multiple optimizers.
7. Weight checkpointing.
8. Model saving.
@@ -49,7 +49,7 @@ Your model.
13. Distribute memory-bound models on multiple GPUs.
14. Give your model hyperparameters parsed from the command line OR a JSON file.
15. Run your model in a dev environment where nothing logs.
-
+
## Usage
To use lightning do 2 things:
1. [Define a trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/trainer_main.py) (which will run ALL your models).
@@ -130,32 +130,32 @@ class My_Model(RootModule):
def __init__(self):
# define model
self.l1 = nn.Linear(200, 10)
-
+
# ---------------
# TRAINING
def training_step(self, data_batch):
x, y = data_batch
y_hat = self.l1(x)
loss = some_loss(y_hat)
-
+
return loss_val, {'train_loss': loss}
-
+
def validation_step(self, data_batch):
x, y = data_batch
y_hat = self.l1(x)
loss = some_loss(y_hat)
-
+
return loss_val, {'val_loss': loss}
-
+
def validation_end(self, outputs):
total_accs = []
-
+
for output in outputs:
total_accs.append(output['val_acc'].item())
-
+
# return a dict
return {'total_acc': np.mean(total_accs)}
-
+
# ---------------
# SAVING
def get_save_dict(self):
@@ -167,7 +167,7 @@ class My_Model(RootModule):
def load_model_specific(self, checkpoint):
# lightning loads for you. Here's your chance to say what you want to load
self.load_state_dict(checkpoint['state_dict'])
-
+
# ---------------
# TRAINING CONFIG
def configure_optimizers(self):
@@ -175,7 +175,7 @@ class My_Model(RootModule):
# lightning will call automatically
optimizer = self.choose_optimizer('adam', self.parameters(), {'lr': self.hparams.learning_rate}, 'optimizer')
return [optimizer]
-
+
@property
def tng_dataloader(self):
return pytorch_dataloader('train')
@@ -187,7 +187,7 @@ class My_Model(RootModule):
@property
def test_dataloader(self):
return pytorch_dataloader('test')
-
+
# ---------------
# MODIFY YOUR COMMAND LINE ARGS
@staticmethod
@@ -206,7 +206,7 @@ class My_Model(RootModule):
| training_step | Called with a batch of data during training | data from your dataloaders | tuple: scalar, dict |
| validation_step | Called with a batch of data during validation | data from your dataloaders | tuple: scalar, dict |
| validation_end | Collate metrics from all validation steps | outputs: array where each item is the output of a validation step | dict: for logging |
-| get_save_dict | called when your model needs to be saved (checkpoints, hpc save, etc...) | None | dict to be saved |
+| get_save_dict | called when your model needs to be saved (checkpoints, hpc save, etc...) | None | dict to be saved |
#### Model training
| Name | Description | Input | Return |
@@ -222,7 +222,7 @@ class My_Model(RootModule):
|---|---|---|---|
| get_save_dict | called when your model needs to be saved (checkpoints, hpc save, etc...) | None | dict to be saved |
| load_model_specific | called when loading a model | checkpoint: dict you created in get_save_dict | dict: modified in whatever way you want |
-
+
## Optional model hooks.
Add these to the model whenever you want to configure training behavior.
diff --git a/imgs/lightning_logo.png b/docs/source/_static/lightning_logo.png
similarity index 100%
rename from imgs/lightning_logo.png
rename to docs/source/_static/lightning_logo.png
diff --git a/demo/basic_trainer.py b/docs/source/examples/basic_trainer.py
similarity index 100%
rename from demo/basic_trainer.py
rename to docs/source/examples/basic_trainer.py
diff --git a/demo/example_model.py b/docs/source/examples/example_model.py
similarity index 100%
rename from demo/example_model.py
rename to docs/source/examples/example_model.py
diff --git a/demo/fully_featured_trainer.py b/docs/source/examples/fully_featured_trainer.py
similarity index 100%
rename from demo/fully_featured_trainer.py
rename to docs/source/examples/fully_featured_trainer.py
diff --git a/imgs/.DS_Store b/imgs/.DS_Store
deleted file mode 100644
index 1d3f4c0ab6..0000000000
Binary files a/imgs/.DS_Store and /dev/null differ
diff --git a/pytorch_lightning/root_module/__init__.py b/pytorch_lightning/root_module/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/pytorch_lightning/utils/__init__.py b/pytorch_lightning/utils/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/setup.py b/setup.py
index 80bb2b1502..334369148d 100755
--- a/setup.py
+++ b/setup.py
@@ -2,12 +2,58 @@
from setuptools import setup, find_packages
-setup(name='pytorch-lightning',
- version='0.0.2',
- description='Rapid research framework',
- author='',
- author_email='',
- url='https://github.com/williamFalcon/pytorch-lightning',
- install_requires=['test-tube', 'torch', 'tqdm'],
- packages=find_packages()
- )
+# https://packaging.python.org/guides/single-sourcing-package-version/
+version = {}
+with open(os.path.join("src", "pytorch-lightning", "__init__.py")) as fp:
+ exec(fp.read(), version)
+
+# http://blog.ionelmc.ro/2014/05/25/python-packaging/
+setup(
+ name="pytorch-lightning",
+ version=version["__version__"],
+ description="The Keras for ML researchers using PyTorch",
+ author="William Falcon",
+ author_email="waf2107@columbia.edu",
+ url="https://github.com/williamFalcon/pytorch-lightning",
+ download_url="https://github.com/williamFalcon/pytorch-lightning",
+ license="MIT",
+ keywords=["deep learning", "pytorch", "AI"],
+ python_requires=">=3.5",
+ install_requires=[
+ "torch",
+ "tqdm",
+ "test-tube",
+ ],
+ extras_require={
+ "dev": [
+ "black ; python_version>='3.6'",
+ "coverage",
+ "isort",
+ "pytest",
+ "pytest-cov<2.6.0",
+ "pycodestyle",
+ "sphinx",
+ "nbsphinx",
+ "ipython>=5.0",
+ "jupyter-client",
+ ]
+ },
+ packages=find_packages("src"),
+ package_dir={"": "src"},
+ entry_points={"console_scripts": ["pytorch-lightning = pytorch-lightning.cli:main"]},
+ classifiers=[
+ "Development Status :: 4 - Beta",
+ "Intended Audience :: Education",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: MIT License",
+ "Operating System :: OS Independent",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.5",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ ],
+ long_description=open("README.md", encoding="utf-8").read(),
+ include_package_data=True,
+ zip_safe=False,
+)
diff --git a/src/pytorch-lightning/__init__.py b/src/pytorch-lightning/__init__.py
new file mode 100644
index 0000000000..7ce8e11283
--- /dev/null
+++ b/src/pytorch-lightning/__init__.py
@@ -0,0 +1,10 @@
+"""
+=================
+pytorch-lightning
+=================
+
+The Keras for ML researchers using PyTorch. More control. Less boilerplate.
+
+"""
+
+__version__ = "0.1.dev0"
diff --git a/__init__.py b/src/pytorch-lightning/models/__init__.py
similarity index 100%
rename from __init__.py
rename to src/pytorch-lightning/models/__init__.py
diff --git a/notebooks/__init__.py b/src/pytorch-lightning/models/model_examples/__init__.py
similarity index 100%
rename from notebooks/__init__.py
rename to src/pytorch-lightning/models/model_examples/__init__.py
diff --git a/src/pytorch-lightning/models/model_examples/bilstm.py b/src/pytorch-lightning/models/model_examples/bilstm.py
new file mode 100644
index 0000000000..f221a9d778
--- /dev/null
+++ b/src/pytorch-lightning/models/model_examples/bilstm.py
@@ -0,0 +1,167 @@
+import torch.nn as nn
+import numpy as np
+
+from test_tube import HyperOptArgumentParser
+import torch
+from torch.autograd import Variable
+from sklearn.metrics import confusion_matrix, f1_score
+from torch.nn import functional as F
+
+
+class BiLSTMPack(nn.Module):
+ """
+ Sample model to show how to define a template
+ """
+ def __init__(self, hparams):
+ # init superclass
+ super(BiLSTMPack, self).__init__(hparams)
+
+ self.hidden = None
+
+ # trigger tag building
+ self.ner_tagset = {'O': 0, 'I-Bio': 1}
+ self.nb_tags = len(self.ner_tagset)
+
+ # build model
+ print('building model...')
+ if hparams.model_load_weights_path is None:
+ self.__build_model()
+ print('model built')
+ else:
+ self = BiLSTMPack.load(hparams.model_load_weights_path, hparams.on_gpu, hparams)
+ print('model loaded from: {}'.format(hparams.model_load_weights_path))
+
+ def __build_model(self):
+ """
+ Layout model
+ :return:
+ """
+ # design the number of final units
+ self.output_dim = self.hparams.nb_lstm_units
+
+ # when it's bidirectional our weights double
+ if self.hparams.bidirectional:
+ self.output_dim *= 2
+
+ # total number of words
+ total_words = len(self.tng_dataloader.dataset.words_token_to_idx)
+
+ # word embeddings
+ self.word_embedding = nn.Embedding(
+ num_embeddings=total_words + 1,
+ embedding_dim=self.hparams.embedding_dim,
+ padding_idx=0
+ )
+
+ # design the LSTM
+ self.lstm = nn.LSTM(
+ self.hparams.embedding_dim,
+ self.hparams.nb_lstm_units,
+ num_layers=self.hparams.nb_lstm_layers,
+ bidirectional=self.hparams.bidirectional,
+ dropout=self.hparams.drop_prob,
+ batch_first=True,
+ )
+
+ # map to tag space
+ self.fc_out = nn.Linear(self.output_dim, self.out_dim)
+ self.hidden_to_tag = nn.Linear(self.output_dim, self.nb_tags)
+
+
+ def init_hidden(self, batch_size):
+
+ # the weights are of the form (nb_layers * 2 if bidirectional, batch_size, nb_lstm_units)
+ mult = 2 if self.hparams.bidirectional else 1
+ hidden_a = torch.randn(self.hparams.nb_layers * mult, batch_size, self.nb_rnn_units)
+ hidden_b = torch.randn(self.hparams.nb_layers * mult, batch_size, self.nb_rnn_units)
+
+ if self.hparams.on_gpu:
+ hidden_a = hidden_a.cuda()
+ hidden_b = hidden_b.cuda()
+
+ hidden_a = Variable(hidden_a)
+ hidden_b = Variable(hidden_b)
+
+ return (hidden_a, hidden_b)
+
+ def forward(self, model_in):
+ # layout data (expand it, etc...)
+ # x = sequences
+ x, seq_lengths = model_in
+ batch_size, seq_len = x.size()
+
+ # reset RNN hidden state
+ self.hidden = self.init_hidden(batch_size)
+
+ # embed
+ x = self.word_embedding(x)
+
+ # run through rnn using packed sequences
+ x = torch.nn.utils.rnn.pack_padded_sequence(x, seq_lengths, batch_first=True)
+ x, self.hidden = self.lstm(x, self.hidden)
+ x, _ = torch.nn.utils.rnn.pad_packed_sequence(x, batch_first=True)
+
+ # if asked for only last state, use the h_n which is the same as out(t=n)
+ if not self.return_sequence:
+ # pull out hidden states
+ # h_n = (nb_directions * nb_layers, batch_size, emb_size)
+ nb_directions = 2 if self.bidirectional else 1
+ (h_n, _) = self.hidden
+
+ # reshape to make indexing easier
+ # forward = 0, backward = 1 (of nb_directions)
+ h_n = h_n.view(self.nb_layers, nb_directions, batch_size, self.nb_rnn_units)
+
+ # pull out last forward
+ forward_h_n = h_n[-1, 0, :, :]
+ x = forward_h_n
+
+ # if bidirectional, also pull out the last hidden of backward network
+ if self.bidirectional:
+ backward_h_n = h_n[-1, 1, :, :]
+ x = torch.cat([forward_h_n, backward_h_n], dim=1)
+
+ # project to tag space
+ x = x.contiguous()
+ x = x.view(-1, self.output_dim)
+ x = self.hidden_to_tag(x)
+
+ return x
+
+ def loss(self, model_out):
+ # cross entropy loss
+ logits, y = model_out
+ y, y_lens = y
+
+ # flatten y and logits
+ y = y.view(-1)
+ logits = logits.view(-1, self.nb_tags)
+
+ # calculate a mask to remove padding tokens
+ mask = (y >= 0).float()
+
+ # count how many tokens we have
+ num_tokens = int(torch.sum(mask).data[0])
+
+ # pick the correct values and mask out
+ logits = logits[range(logits.shape[0]), y] * mask
+
+ # compute the ce loss
+ ce_loss = -torch.sum(logits)/num_tokens
+
+ return ce_loss
+
+ def pull_out_last_embedding(self, x, seq_lengths, batch_size, on_gpu):
+ # grab only the last activations from the non-padded ouput
+ x_last = torch.zeros([batch_size, 1, x.size(-1)])
+ for i, seq_len in enumerate(seq_lengths):
+ x_last[i, :, :] = x[i, seq_len-1, :]
+
+ # put on gpu when requested
+ if on_gpu:
+ x_last = x_last.cuda()
+
+ # turn into torch var
+ x_last = Variable(x_last)
+
+ return x_last
\ No newline at end of file
diff --git a/pytorch_lightning/__init__.py b/src/pytorch-lightning/models/sample_model_template/__init__.py
similarity index 100%
rename from pytorch_lightning/__init__.py
rename to src/pytorch-lightning/models/sample_model_template/__init__.py
diff --git a/pytorch_lightning/models/sample_model_template/model_template.py b/src/pytorch-lightning/models/sample_model_template/model_template.py
similarity index 100%
rename from pytorch_lightning/models/sample_model_template/model_template.py
rename to src/pytorch-lightning/models/sample_model_template/model_template.py
diff --git a/pytorch_lightning/models/trainer.py b/src/pytorch-lightning/models/trainer.py
similarity index 100%
rename from pytorch_lightning/models/trainer.py
rename to src/pytorch-lightning/models/trainer.py
diff --git a/pytorch_lightning/models/__init__.py b/src/pytorch-lightning/root_module/__init__.py
similarity index 100%
rename from pytorch_lightning/models/__init__.py
rename to src/pytorch-lightning/root_module/__init__.py
diff --git a/pytorch_lightning/root_module/grads.py b/src/pytorch-lightning/root_module/grads.py
similarity index 100%
rename from pytorch_lightning/root_module/grads.py
rename to src/pytorch-lightning/root_module/grads.py
diff --git a/pytorch_lightning/root_module/hooks.py b/src/pytorch-lightning/root_module/hooks.py
similarity index 100%
rename from pytorch_lightning/root_module/hooks.py
rename to src/pytorch-lightning/root_module/hooks.py
diff --git a/pytorch_lightning/root_module/memory.py b/src/pytorch-lightning/root_module/memory.py
similarity index 100%
rename from pytorch_lightning/root_module/memory.py
rename to src/pytorch-lightning/root_module/memory.py
diff --git a/pytorch_lightning/root_module/model_saving.py b/src/pytorch-lightning/root_module/model_saving.py
similarity index 100%
rename from pytorch_lightning/root_module/model_saving.py
rename to src/pytorch-lightning/root_module/model_saving.py
diff --git a/pytorch_lightning/root_module/optimization.py b/src/pytorch-lightning/root_module/optimization.py
similarity index 100%
rename from pytorch_lightning/root_module/optimization.py
rename to src/pytorch-lightning/root_module/optimization.py
diff --git a/pytorch_lightning/root_module/root_module.py b/src/pytorch-lightning/root_module/root_module.py
similarity index 100%
rename from pytorch_lightning/root_module/root_module.py
rename to src/pytorch-lightning/root_module/root_module.py
diff --git a/pytorch_lightning/trainer_main.py b/src/pytorch-lightning/trainer_main.py
similarity index 100%
rename from pytorch_lightning/trainer_main.py
rename to src/pytorch-lightning/trainer_main.py
diff --git a/pytorch_lightning/models/sample_model_template/__init__.py b/src/pytorch-lightning/utils/__init__.py
similarity index 100%
rename from pytorch_lightning/models/sample_model_template/__init__.py
rename to src/pytorch-lightning/utils/__init__.py
diff --git a/pytorch_lightning/utils/arg_parse.py b/src/pytorch-lightning/utils/arg_parse.py
similarity index 100%
rename from pytorch_lightning/utils/arg_parse.py
rename to src/pytorch-lightning/utils/arg_parse.py
diff --git a/pytorch_lightning/utils/embeddings.py b/src/pytorch-lightning/utils/embeddings.py
similarity index 100%
rename from pytorch_lightning/utils/embeddings.py
rename to src/pytorch-lightning/utils/embeddings.py
diff --git a/pytorch_lightning/utils/plotting.py b/src/pytorch-lightning/utils/plotting.py
similarity index 100%
rename from pytorch_lightning/utils/plotting.py
rename to src/pytorch-lightning/utils/plotting.py
diff --git a/pytorch_lightning/utils/pt_callbacks.py b/src/pytorch-lightning/utils/pt_callbacks.py
similarity index 100%
rename from pytorch_lightning/utils/pt_callbacks.py
rename to src/pytorch-lightning/utils/pt_callbacks.py
diff --git a/src/pytorch_lightning/__init__.py b/src/pytorch_lightning/__init__.py
new file mode 100644
index 0000000000..7ce8e11283
--- /dev/null
+++ b/src/pytorch_lightning/__init__.py
@@ -0,0 +1,10 @@
+"""
+=================
+pytorch-lightning
+=================
+
+The Keras for ML researchers using PyTorch. More control. Less boilerplate.
+
+"""
+
+__version__ = "0.1.dev0"
diff --git a/tests/README.md b/tests/README.md
deleted file mode 100644
index 18c7a537c8..0000000000
--- a/tests/README.md
+++ /dev/null
@@ -1,65 +0,0 @@
-# Testing setup
-
-## A. Enable CircleCI for your project
-1. Integrate CircleCI by clicking "Set up Project" at [this link](https://circleci.com/add-projects/gh/NextGenVest).
-
-## B. Add your own tests
-1. In the /tests, emulate exactly the folder structure for your module found under /bot_seed
-2. To create a test for file ```/bot_seed/folder/example.py```:
- - create the file ```/tests/folder/example_test.py```
- - notice the **_test**
- - notice the mirror path under **/tests**
-
-3. Your ```example_test.py``` file should have these main components
-
-```python
-# example.py
-
-def function_i_want_to_test(x):
- return x*2
-
-def square(x):
- return x*x
-
-```
-
-```python
-# example_test.py
-
-import pytest
-
-# do whatever imports you need
-from app.bot_seed.folder.example import function_i_want_to_test, square
-
-def test_function_i_want_to_test():
- answer = function_i_want_to_test(4)
- assert answer == 8
-
-# -----------------------------------
-# Your function must start with test_
-# -----------------------------------
-def test_square():
- answer = square(3)
- assert answer == 9
-
-# -----------------------------------
-# boilerplate (link this file to pytest)
-# -----------------------------------
-if __name__ == '__main__':
- pytest.main([__file__])
-```
-
-## C. Add build passing badge
-1. Create a CircleCI status token:
- - Go here: https://circleci.com/gh/NextGenVest/your-project-name/edit#api
- - Click create token
- - Select status
- - Type "badge status"
-
-2. Get a copy of the markdown code:
- - Go here: https://circleci.com/gh/NextGenVest/your-project-name/edit#badges
- - Select master
- - Select "badge status" token
- - Select image URL
- - Copy the image url link and change the html at the top of the root README.md file for your project
-
\ No newline at end of file
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tests/research_proj/__init__.py b/tests/research_proj/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tests/research_proj/sample_model_template/__init__.py b/tests/research_proj/sample_model_template/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tests/research_proj/sample_model_template/model_template_test.py b/tests/research_proj/sample_model_template/model_template_test.py
deleted file mode 100644
index 1ab841611b..0000000000
--- a/tests/research_proj/sample_model_template/model_template_test.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import pytest
-
-"""
-Example test to show how to add a test for anything in the project.
-Look at the README for more instructions
-"""
-
-
-def test_cube():
- assert 27 == 27
-
-if __name__ == '__main__':
- pytest.main([__file__])