ml-finance-python
python scripts for finance machine learning
git clone https://9o.is/git/ml-finance-python.git
restricted_boltzmann_machine.py
(3357B)
1 import logging
2 import numpy as np
3 import progressbar
4
5 from mlfromscratch.utils.misc import bar_widgets
6 from mlfromscratch.utils import batch_iterator
7 from mlfromscratch.deep_learning.activation_functions import Sigmoid
8
9 sigmoid = Sigmoid()
10
11 class RBM():
12 """Bernoulli Restricted Boltzmann Machine (RBM)
13
14 Parameters:
15 -----------
16 n_hidden: int:
17 The number of processing nodes (neurons) in the hidden layer.
18 learning_rate: float
19 The step length that will be used when updating the weights.
20 batch_size: int
21 The size of the mini-batch used to calculate each weight update.
22 n_iterations: float
23 The number of training iterations the algorithm will tune the weights for.
24
25 Reference:
26 A Practical Guide to Training Restricted Boltzmann Machines
27 URL: https://www.cs.toronto.edu/~hinton/absps/guideTR.pdf
28 """
29 def __init__(self, n_hidden=128, learning_rate=0.1, batch_size=10, n_iterations=100):
30 self.n_iterations = n_iterations
31 self.batch_size = batch_size
32 self.lr = learning_rate
33 self.n_hidden = n_hidden
34 self.progressbar = progressbar.ProgressBar(widgets=bar_widgets)
35
36 def _initialize_weights(self, X):
37 n_visible = X.shape[1]
38 self.W = np.random.normal(scale=0.1, size=(n_visible, self.n_hidden))
39 self.v0 = np.zeros(n_visible) # Bias visible
40 self.h0 = np.zeros(self.n_hidden) # Bias hidden
41
42 def fit(self, X, y=None):
43 '''Contrastive Divergence training procedure'''
44
45 self._initialize_weights(X)
46
47 self.training_errors = []
48 self.training_reconstructions = []
49 for _ in self.progressbar(range(self.n_iterations)):
50 batch_errors = []
51 for batch in batch_iterator(X, batch_size=self.batch_size):
52 # Positive phase
53 positive_hidden = sigmoid(batch.dot(self.W) + self.h0)
54 hidden_states = self._sample(positive_hidden)
55 positive_associations = batch.T.dot(positive_hidden)
56
57 # Negative phase
58 negative_visible = sigmoid(hidden_states.dot(self.W.T) + self.v0)
59 negative_visible = self._sample(negative_visible)
60 negative_hidden = sigmoid(negative_visible.dot(self.W) + self.h0)
61 negative_associations = negative_visible.T.dot(negative_hidden)
62
63 self.W += self.lr * (positive_associations - negative_associations)
64 self.h0 += self.lr * (positive_hidden.sum(axis=0) - negative_hidden.sum(axis=0))
65 self.v0 += self.lr * (batch.sum(axis=0) - negative_visible.sum(axis=0))
66
67 batch_errors.append(np.mean((batch - negative_visible) ** 2))
68
69 self.training_errors.append(np.mean(batch_errors))
70 # Reconstruct a batch of images from the training set
71 idx = np.random.choice(range(X.shape[0]), self.batch_size)
72 self.training_reconstructions.append(self.reconstruct(X[idx]))
73
74 def _sample(self, X):
75 return X > np.random.random_sample(size=X.shape)
76
77 def reconstruct(self, X):
78 positive_hidden = sigmoid(X.dot(self.W) + self.h0)
79 hidden_states = self._sample(positive_hidden)
80 negative_visible = sigmoid(hidden_states.dot(self.W.T) + self.v0)
81 return negative_visible
82