ml-finance-python

xgboost.py

(3726B)

 from __future__ import division, print_function
 import numpy as np
 import progressbar
 
 from mlfromscratch.utils import train_test_split, standardize, to_categorical, normalize
 from mlfromscratch.utils import mean_squared_error, accuracy_score
 from mlfromscratch.supervised_learning import XGBoostRegressionTree
 from mlfromscratch.deep_learning.activation_functions import Sigmoid
 from mlfromscratch.utils.misc import bar_widgets
 from mlfromscratch.utils import Plot
 
 
 class LogisticLoss():
     def __init__(self):
         sigmoid = Sigmoid()
         self.log_func = sigmoid
         self.log_grad = sigmoid.gradient
 
     def loss(self, y, y_pred):
         y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
         p = self.log_func(y_pred)
         return y * np.log(p) + (1 - y) * np.log(1 - p)
 
     # gradient w.r.t y_pred
     def gradient(self, y, y_pred):
         p = self.log_func(y_pred)
         return -(y - p)
 
     # w.r.t y_pred
     def hess(self, y, y_pred):
         p = self.log_func(y_pred)
         return p * (1 - p)
 
 
 class XGBoost(object):
     """The XGBoost classifier.
 
     Reference: http://xgboost.readthedocs.io/en/latest/model.html
 
     Parameters:
     -----------
     n_estimators: int
         The number of classification trees that are used.
     learning_rate: float
         The step length that will be taken when following the negative gradient during
         training.
     min_samples_split: int
         The minimum number of samples needed to make a split when building a tree.
     min_impurity: float
         The minimum impurity required to split the tree further. 
     max_depth: int
         The maximum depth of a tree.
     """
     def __init__(self, n_estimators=200, learning_rate=0.001, min_samples_split=2,
                  min_impurity=1e-7, max_depth=2):
         self.n_estimators = n_estimators            # Number of trees
         self.learning_rate = learning_rate          # Step size for weight update
         self.min_samples_split = min_samples_split  # The minimum n of sampels to justify split
         self.min_impurity = min_impurity              # Minimum variance reduction to continue
         self.max_depth = max_depth                  # Maximum depth for tree
 
         self.bar = progressbar.ProgressBar(widgets=bar_widgets)
         
         # Log loss for classification
         self.loss = LogisticLoss()
 
         # Initialize regression trees
         self.trees = []
         for _ in range(n_estimators):
             tree = XGBoostRegressionTree(
                     min_samples_split=self.min_samples_split,
                     min_impurity=min_impurity,
                     max_depth=self.max_depth,
                     loss=self.loss)
 
             self.trees.append(tree)
 
     def fit(self, X, y):
         y = to_categorical(y)
 
         y_pred = np.zeros(np.shape(y))
         for i in self.bar(range(self.n_estimators)):
             tree = self.trees[i]
             y_and_pred = np.concatenate((y, y_pred), axis=1)
             tree.fit(X, y_and_pred)
             update_pred = tree.predict(X)
 
             y_pred -= np.multiply(self.learning_rate, update_pred)
 
     def predict(self, X):
         y_pred = None
         # Make predictions
         for tree in self.trees:
             # Estimate gradient and update prediction
             update_pred = tree.predict(X)
             if y_pred is None:
                 y_pred = np.zeros_like(update_pred)
             y_pred -= np.multiply(self.learning_rate, update_pred)
 
         # Turn into probability distribution (Softmax)
         y_pred = np.exp(y_pred) / np.sum(np.exp(y_pred), axis=1, keepdims=True)
         # Set label to the value that maximizes probability
         y_pred = np.argmax(y_pred, axis=1)
         return y_pred