ml-finance-python

python scripts for finance machine learning
git clone https://9o.is/git/ml-finance-python.git
activation_functions.py

(1992B)
      1 import numpy as np
      2 
      3 # Collection of activation functions
      4 # Reference: https://en.wikipedia.org/wiki/Activation_function
      5 
      6 class Sigmoid():
      7     def __call__(self, x):
      8         return 1 / (1 + np.exp(-x))
      9 
     10     def gradient(self, x):
     11         return self.__call__(x) * (1 - self.__call__(x))
     12 
     13 class Softmax():
     14     def __call__(self, x):
     15         e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
     16         return e_x / np.sum(e_x, axis=-1, keepdims=True)
     17 
     18     def gradient(self, x):
     19         p = self.__call__(x)
     20         return p * (1 - p)
     21 
     22 class TanH():
     23     def __call__(self, x):
     24         return 2 / (1 + np.exp(-2*x)) - 1
     25 
     26     def gradient(self, x):
     27         return 1 - np.power(self.__call__(x), 2)
     28 
     29 class ReLU():
     30     def __call__(self, x):
     31         return np.where(x >= 0, x, 0)
     32 
     33     def gradient(self, x):
     34         return np.where(x >= 0, 1, 0)
     35 
     36 class LeakyReLU():
     37     def __init__(self, alpha=0.2):
     38         self.alpha = alpha
     39 
     40     def __call__(self, x):
     41         return np.where(x >= 0, x, self.alpha * x)
     42 
     43     def gradient(self, x):
     44         return np.where(x >= 0, 1, self.alpha)
     45 
     46 class ELU():
     47     def __init__(self, alpha=0.1):
     48         self.alpha = alpha 
     49 
     50     def __call__(self, x):
     51         return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))
     52 
     53     def gradient(self, x):
     54         return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)
     55 
     56 class SELU():
     57     # Reference : https://arxiv.org/abs/1706.02515,
     58     # https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
     59     def __init__(self):
     60         self.alpha = 1.6732632423543772848170429916717
     61         self.scale = 1.0507009873554804934193349852946 
     62 
     63     def __call__(self, x):
     64         return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))
     65 
     66     def gradient(self, x):
     67         return self.scale * np.where(x >= 0.0, 1, self.alpha * np.exp(x))
     68 
     69 class SoftPlus():
     70     def __call__(self, x):
     71         return np.log(1 + np.exp(x))
     72 
     73     def gradient(self, x):
     74         return 1 / (1 + np.exp(-x))
     75