ml-finance-python
python scripts for finance machine learning
git clone https://9o.is/git/ml-finance-python.git
06_ml_for_trading.ipynb
(22498B)
1 {
2 "cells": [
3 {
4 "cell_type": "markdown",
5 "metadata": {},
6 "source": [
7 "# ML for Trading: How to run an ML algorithm on Quantopian"
8 ]
9 },
10 {
11 "cell_type": "markdown",
12 "metadata": {},
13 "source": [
14 "The code in this notebook is written for the Quantopian Research Platform and uses the 'Algorithms' rather than the 'Research' option we used before."
15 ]
16 },
17 {
18 "cell_type": "markdown",
19 "metadata": {},
20 "source": [
21 "To run it, you need to have a free Quantopian account, create a new algorithm and copy the content to the online development environment."
22 ]
23 },
24 {
25 "cell_type": "markdown",
26 "metadata": {},
27 "source": [
28 "## Imports & Settings"
29 ]
30 },
31 {
32 "cell_type": "markdown",
33 "metadata": {},
34 "source": [
35 "### Quantopian Libraries"
36 ]
37 },
38 {
39 "cell_type": "code",
40 "execution_count": null,
41 "metadata": {},
42 "outputs": [],
43 "source": [
44 "from quantopian.algorithm import attach_pipeline, pipeline_output, order_optimal_portfolio\n",
45 "from quantopian.pipeline import Pipeline, factors, filters, classifiers\n",
46 "from quantopian.pipeline.data.builtin import USEquityPricing\n",
47 "from quantopian.pipeline.data import Fundamentals\n",
48 "from quantopian.pipeline.data.psychsignal import stocktwits\n",
49 "from quantopian.pipeline.factors import (Latest, \n",
50 " CustomFactor, \n",
51 " SimpleMovingAverage, \n",
52 " AverageDollarVolume, \n",
53 " Returns, \n",
54 " RSI, \n",
55 " SimpleBeta, \n",
56 " MovingAverageConvergenceDivergenceSignal as MACD)\n",
57 "from quantopian.pipeline.filters import QTradableStocksUS\n",
58 "from quantopian.pipeline.experimental import risk_loading_pipeline, Size, Momentum, Volatility, Value, ShortTermReversal\n",
59 "\n",
60 "import quantopian.optimize as opt\n",
61 "from quantopian.optimize.experimental import RiskModelExposure"
62 ]
63 },
64 {
65 "cell_type": "markdown",
66 "metadata": {},
67 "source": [
68 "### Other Python Libraries"
69 ]
70 },
71 {
72 "cell_type": "code",
73 "execution_count": null,
74 "metadata": {},
75 "outputs": [],
76 "source": [
77 "from scipy.stats import spearmanr\n",
78 "import talib\n",
79 "import pandas as pd\n",
80 "import numpy as np\n",
81 "from time import time\n",
82 "from collections import OrderedDict\n",
83 "\n",
84 "from scipy import stats\n",
85 "from sklearn import linear_model, preprocessing, metrics, cross_validation\n",
86 "from sklearn.pipeline import make_pipeline"
87 ]
88 },
89 {
90 "cell_type": "markdown",
91 "metadata": {},
92 "source": [
93 "### Strategy Positions"
94 ]
95 },
96 {
97 "cell_type": "code",
98 "execution_count": null,
99 "metadata": {},
100 "outputs": [],
101 "source": [
102 "# strategy parameters\n",
103 "N_POSITIONS = 100 # Will be split 50% long and 50% short\n",
104 "TRAINING_PERIOD = 126 # past periods for training\n",
105 "HOLDING_PERIOD = 5 # predict returns N days into the future\n",
106 "\n",
107 "# How often to trade, for daily, alternative is date_rules.every_day()\n",
108 "TRADE_FREQ = date_rules.week_start()"
109 ]
110 },
111 {
112 "cell_type": "markdown",
113 "metadata": {},
114 "source": [
115 "### Custom Universe"
116 ]
117 },
118 {
119 "cell_type": "markdown",
120 "metadata": {},
121 "source": [
122 "We define a custom universe to limit duration of training."
123 ]
124 },
125 {
126 "cell_type": "code",
127 "execution_count": null,
128 "metadata": {},
129 "outputs": [],
130 "source": [
131 "def Q250US():\n",
132 " \"\"\"Define custom universe\"\"\"\n",
133 " return filters.make_us_equity_universe(\n",
134 " target_size=250,\n",
135 " rankby=factors.AverageDollarVolume(window_length=200),\n",
136 " mask=filters.default_us_equity_universe_mask(),\n",
137 " groupby=classifiers.fundamentals.Sector(),\n",
138 " max_group_weight=0.3,\n",
139 " smoothing_func=lambda f: f.downsample('month_start'),\n",
140 " )"
141 ]
142 },
143 {
144 "cell_type": "markdown",
145 "metadata": {},
146 "source": [
147 "## Create Alpha Factors"
148 ]
149 },
150 {
151 "cell_type": "code",
152 "execution_count": null,
153 "metadata": {},
154 "outputs": [],
155 "source": [
156 "def make_alpha_factors():\n",
157 " \n",
158 " def PriceToSalesTTM():\n",
159 " \"\"\"Last closing price divided by sales per share\"\"\" \n",
160 " return Fundamentals.ps_ratio.latest\n",
161 "\n",
162 " def PriceToEarningsTTM():\n",
163 " \"\"\"Closing price divided by earnings per share (EPS)\"\"\"\n",
164 " return Fundamentals.pe_ratio.latest\n",
165 " \n",
166 " def DividendYield():\n",
167 " \"\"\"Dividends per share divided by closing price\"\"\"\n",
168 " return Fundamentals.trailing_dividend_yield.latest\n",
169 " \n",
170 " def Capex_To_Cashflows():\n",
171 " return (Fundamentals.capital_expenditure.latest * 4.) / \\\n",
172 " (Fundamentals.free_cash_flow.latest * 4.)\n",
173 " \n",
174 " def EBITDA_Yield():\n",
175 " return (Fundamentals.ebitda.latest * 4.) / \\\n",
176 " USEquityPricing.close.latest \n",
177 "\n",
178 " def EBIT_To_Assets():\n",
179 " return (Fundamentals.ebit.latest * 4.) / \\\n",
180 " Fundamentals.total_assets.latest\n",
181 " \n",
182 " def Return_On_Total_Invest_Capital():\n",
183 " return Fundamentals.roic.latest\n",
184 " \n",
185 " class Mean_Reversion_1M(CustomFactor):\n",
186 " inputs = [Returns(window_length=21)]\n",
187 " window_length = 252\n",
188 "\n",
189 " def compute(self, today, assets, out, monthly_rets):\n",
190 " out[:] = (monthly_rets[-1] - np.nanmean(monthly_rets, axis=0)) / \\\n",
191 " np.nanstd(monthly_rets, axis=0)\n",
192 " \n",
193 " def MACD_Signal():\n",
194 " return MACD(fast_period=12, slow_period=26, signal_period=9)\n",
195 " \n",
196 " def Net_Income_Margin():\n",
197 " return Fundamentals.net_margin.latest \n",
198 "\n",
199 " def Operating_Cashflows_To_Assets():\n",
200 " return (Fundamentals.operating_cash_flow.latest * 4.) / \\\n",
201 " Fundamentals.total_assets.latest\n",
202 "\n",
203 " def Price_Momentum_3M():\n",
204 " return Returns(window_length=63)\n",
205 " \n",
206 " class Price_Oscillator(CustomFactor):\n",
207 " inputs = [USEquityPricing.close]\n",
208 " window_length = 252\n",
209 "\n",
210 " def compute(self, today, assets, out, close):\n",
211 " four_week_period = close[-20:]\n",
212 " out[:] = (np.nanmean(four_week_period, axis=0) /\n",
213 " np.nanmean(close, axis=0)) - 1.\n",
214 " \n",
215 " def Returns_39W():\n",
216 " return Returns(window_length=215)\n",
217 " \n",
218 " class Vol_3M(CustomFactor):\n",
219 " inputs = [Returns(window_length=2)]\n",
220 " window_length = 63\n",
221 "\n",
222 " def compute(self, today, assets, out, rets):\n",
223 " out[:] = np.nanstd(rets, axis=0)\n",
224 " \n",
225 " def Working_Capital_To_Assets():\n",
226 " return Fundamentals.working_capital.latest / Fundamentals.total_assets.latest\n",
227 " \n",
228 " def sentiment():\n",
229 " return SimpleMovingAverage(inputs=[stocktwits.bull_minus_bear],\n",
230 " window_length=5).rank(mask=universe)\n",
231 " \n",
232 " class AdvancedMomentum(CustomFactor):\n",
233 " \"\"\" Momentum factor \"\"\"\n",
234 " inputs = [USEquityPricing.close,\n",
235 " Returns(window_length=126)]\n",
236 " window_length = 252\n",
237 "\n",
238 " def compute(self, today, assets, out, prices, returns):\n",
239 " out[:] = ((prices[-21] - prices[-252])/prices[-252] -\n",
240 " (prices[-1] - prices[-21])/prices[-21]) / np.nanstd(returns, axis=0)\n",
241 " \n",
242 " def SPY_Beta():\n",
243 " return SimpleBeta(target=sid(8554), regression_length=252)\n",
244 "\n",
245 " return {\n",
246 " 'Price to Sales': PriceToSalesTTM,\n",
247 " 'PE Ratio': PriceToEarningsTTM,\n",
248 " 'Dividend Yield': DividendYield,\n",
249 " # 'Capex to Cashflows': Capex_To_Cashflows,\n",
250 " # 'EBIT to Assets': EBIT_To_Assets,\n",
251 " # 'EBITDA Yield': EBITDA_Yield, \n",
252 " 'MACD Signal Line': MACD_Signal,\n",
253 " 'Mean Reversion 1M': Mean_Reversion_1M,\n",
254 " 'Net Income Margin': Net_Income_Margin, \n",
255 " # 'Operating Cashflows to Assets': Operating_Cashflows_To_Assets,\n",
256 " 'Price Momentum 3M': Price_Momentum_3M,\n",
257 " 'Price Oscillator': Price_Oscillator,\n",
258 " # 'Return on Invested Capital': Return_On_Total_Invest_Capital,\n",
259 " '39 Week Returns': Returns_39W,\n",
260 " 'Vol 3M': Vol_3M,\n",
261 " 'SPY_Beta': SPY_Beta,\n",
262 " 'Advanced Momentum': AdvancedMomentum,\n",
263 " 'Size': Size,\n",
264 " 'Volatitility': Volatility,\n",
265 " 'Value': Value,\n",
266 " 'Short-Term Reversal': ShortTermReversal,\n",
267 " 'Momentum': Momentum,\n",
268 " # 'Materials': materials,\n",
269 " # 'Consumer Discretionary': consumer_discretionary,\n",
270 " # 'Financials': financials,\n",
271 " # 'Real Estate': real_estate,\n",
272 " # 'Consumer Staples': consumer_staples,\n",
273 " # 'Healthcare': health_care,\n",
274 " # 'Utilities': utilities,\n",
275 " # 'Telecom ': telecom,\n",
276 " # 'Energy': energy,\n",
277 " # 'Industrials': industrials,\n",
278 " # 'Technology': technology\n",
279 " }"
280 ]
281 },
282 {
283 "cell_type": "markdown",
284 "metadata": {},
285 "source": [
286 "## Custom Machine Learning Factor"
287 ]
288 },
289 {
290 "cell_type": "markdown",
291 "metadata": {},
292 "source": [
293 "Here we define a Machine Learning factor which trains a model and predicts forward returns "
294 ]
295 },
296 {
297 "cell_type": "code",
298 "execution_count": null,
299 "metadata": {},
300 "outputs": [],
301 "source": [
302 "class ML(CustomFactor):\n",
303 " init = False\n",
304 "\n",
305 " def compute(self, today, assets, out, returns, *inputs):\n",
306 " \"\"\"Train the model using \n",
307 " - shifted returns as target, and \n",
308 " - factors in a list of inputs as features; \n",
309 " each factor contains a 2-D array of shape [time x stocks]\n",
310 " \"\"\"\n",
311 " \n",
312 " if (not self.init) or today.strftime('%A') == 'Monday':\n",
313 " # train on first day then subsequent Mondays (memory)\n",
314 " # get features\n",
315 " features = pd.concat([pd.DataFrame(data, columns=assets).stack().to_frame(i) \n",
316 " for i, data in enumerate(inputs)], axis=1)\n",
317 " \n",
318 " # shift returns and align features\n",
319 " target = (pd.DataFrame(returns, columns=assets)\n",
320 " .shift(-HOLDING_PERIOD)\n",
321 " .dropna(how='all')\n",
322 " .stack())\n",
323 " target.index.rename(['date', 'asset'], inplace=True)\n",
324 " features = features.reindex(target.index)\n",
325 " \n",
326 " # finalize features \n",
327 " features = (pd.get_dummies(features\n",
328 " .assign(asset=features\n",
329 " .index.get_level_values('asset')), \n",
330 " columns=['asset'], \n",
331 " sparse=True))\n",
332 " \n",
333 "\n",
334 " # train the model\n",
335 " self.model_pipe = make_pipeline(preprocessing.Imputer(),\n",
336 " preprocessing.MinMaxScaler(),\n",
337 " linear_model.LinearRegression())\n",
338 "\n",
339 " \n",
340 " # run pipeline and train model\n",
341 " self.model_pipe.fit(X=features, y=target)\n",
342 " self.assets = assets # keep track of assets in model\n",
343 " self.init = True\n",
344 "\n",
345 " # predict most recent factor values\n",
346 " features = pd.DataFrame({i: d[-1] for i, d in enumerate(inputs)}, index=assets)\n",
347 " features = features.reindex(index=self.assets).assign(asset=self.assets)\n",
348 " features = pd.get_dummies(features, columns=['asset']) \n",
349 " \n",
350 " preds = self.model_pipe.predict(features)\n",
351 " out[:] = pd.Series(preds, index=self.assets).reindex(index=assets)"
352 ]
353 },
354 {
355 "cell_type": "markdown",
356 "metadata": {},
357 "source": [
358 "## Create Factor Pipeline"
359 ]
360 },
361 {
362 "cell_type": "markdown",
363 "metadata": {},
364 "source": [
365 "Create pipeline with predictive factors and target returns"
366 ]
367 },
368 {
369 "cell_type": "code",
370 "execution_count": null,
371 "metadata": {},
372 "outputs": [],
373 "source": [
374 "def make_ml_pipeline(alpha_factors, universe, lookback=21, lookahead=5):\n",
375 " \"\"\"Create pipeline with predictive factors and target returns\"\"\"\n",
376 " \n",
377 " # set up pipeline\n",
378 " pipe = OrderedDict()\n",
379 " \n",
380 " # Returns over lookahead days.\n",
381 " pipe['Returns'] = Returns(inputs=[USEquityPricing.open],\n",
382 " mask=universe, \n",
383 " window_length=lookahead + 1)\n",
384 " \n",
385 " # Rank alpha factors:\n",
386 " pipe.update({name: f().rank(mask=universe) \n",
387 " for name, f in alpha_factors.items()})\n",
388 " \n",
389 " # ML factor gets `lookback` datapoints on each factor\n",
390 " pipe['ML'] = ML(inputs=pipe.values(),\n",
391 " window_length=lookback + 1, \n",
392 " mask=universe)\n",
393 " \n",
394 " return Pipeline(columns=pipe, screen=universe) "
395 ]
396 },
397 {
398 "cell_type": "markdown",
399 "metadata": {},
400 "source": [
401 "## Define Algorithm"
402 ]
403 },
404 {
405 "cell_type": "code",
406 "execution_count": null,
407 "metadata": {},
408 "outputs": [],
409 "source": [
410 "def initialize(context):\n",
411 " \"\"\"\n",
412 " Called once at the start of the algorithm.\n",
413 " \"\"\" \n",
414 " set_slippage(slippage.FixedSlippage(spread=0.00))\n",
415 " set_commission(commission.PerShare(cost=0, min_trade_cost=0))\n",
416 " \n",
417 " schedule_function(rebalance_portfolio, \n",
418 " TRADE_FREQ,\n",
419 " time_rules.market_open(minutes=1))\n",
420 " \n",
421 " # Record tracking variables at the end of each day.\n",
422 " schedule_function(log_metrics, \n",
423 " date_rules.every_day(),\n",
424 " time_rules.market_close())\n",
425 "\n",
426 " # Set up universe\n",
427 " # base_universe = AverageDollarVolume(window_length=63, mask=QTradableStocksUS()).percentile_between(80, 100) \n",
428 " universe = AverageDollarVolume(window_length=63, mask=QTradableStocksUS()).percentile_between(40, 60)\n",
429 " \n",
430 " # create alpha factors and machine learning pipline\n",
431 " ml_pipeline = make_ml_pipeline(alpha_factors=make_alpha_factors(),\n",
432 " universe=universe, \n",
433 " lookback=TRAINING_PERIOD,\n",
434 " lookahead=HOLDING_PERIOD)\n",
435 " attach_pipeline(ml_pipeline, 'alpha_model')\n",
436 "\n",
437 " attach_pipeline(risk_loading_pipeline(), 'risk_loading_pipeline')\n",
438 "\n",
439 " context.past_predictions = {}\n",
440 " context.realized_rmse = 0\n",
441 " context.realized_ic = 0\n",
442 " context.long_short_spread = 0"
443 ]
444 },
445 {
446 "cell_type": "markdown",
447 "metadata": {},
448 "source": [
449 "## Evaluate Model"
450 ]
451 },
452 {
453 "cell_type": "markdown",
454 "metadata": {},
455 "source": [
456 "Evaluate model performance using past predictions on hold-out data"
457 ]
458 },
459 {
460 "cell_type": "code",
461 "execution_count": null,
462 "metadata": {},
463 "outputs": [],
464 "source": [
465 "def evaluate_past_predictions(context):\n",
466 " \"\"\"Evaluate model performance using past predictions on hold-out data\"\"\"\n",
467 " # A day has passed, shift days and drop old ones\n",
468 " context.past_predictions = {k-1: v for k, v in context.past_predictions.items() if k-1 >= 0}\n",
469 "\n",
470 " if 0 in context.past_predictions:\n",
471 " # Past predictions for the current day exist, so we can use todays' n-back returns to evaluate them\n",
472 " returns = pipeline_output('alpha_model')['Returns'].to_frame('returns')\n",
473 " \n",
474 " df = (context\n",
475 " .past_predictions[0]\n",
476 " .to_frame('predictions')\n",
477 " .join(returns, how='inner')\n",
478 " .dropna())\n",
479 "\n",
480 " # Compute performance metrics\n",
481 " context.realized_rmse = metrics.mean_squared_error(y_true=df['returns'], y_pred=df.predictions)\n",
482 " context.realized_ic, _ = spearmanr(df['returns'], df.predictions)\n",
483 " log.info('rmse {:.2%} | ic {:.2%}'.format(context.realized_rmse, context.realized_ic))\n",
484 " \n",
485 " long_rets = df.loc[df.predictions >= df.predictions.median(), 'returns'].mean()\n",
486 " short_rets = df.loc[df.predictions < df.predictions.median(), 'returns'].mean()\n",
487 " context.long_short_spread = (long_rets - short_rets) * 100\n",
488 " \n",
489 " # Store current predictions\n",
490 " context.past_predictions[HOLDING_PERIOD] = context.predictions\n",
491 " "
492 ]
493 },
494 {
495 "cell_type": "markdown",
496 "metadata": {},
497 "source": [
498 "## Algo Execution"
499 ]
500 },
501 {
502 "cell_type": "markdown",
503 "metadata": {},
504 "source": [
505 "### Prepare Trades"
506 ]
507 },
508 {
509 "cell_type": "code",
510 "execution_count": null,
511 "metadata": {},
512 "outputs": [],
513 "source": [
514 "def before_trading_start(context, data):\n",
515 " \"\"\"\n",
516 " Called every day before market open.\n",
517 " \"\"\"\n",
518 " context.predictions = pipeline_output('alpha_model')['ML']\n",
519 " context.predictions.index.rename(['date', 'equity'], inplace=True)\n",
520 " context.risk_loading_pipeline = pipeline_output('risk_loading_pipeline') \n",
521 " evaluate_past_predictions(context)"
522 ]
523 },
524 {
525 "cell_type": "markdown",
526 "metadata": {},
527 "source": [
528 "### Rebalance"
529 ]
530 },
531 {
532 "cell_type": "code",
533 "execution_count": null,
534 "metadata": {},
535 "outputs": [],
536 "source": [
537 "def rebalance_portfolio(context, data):\n",
538 " \"\"\"\n",
539 " Execute orders according to our schedule_function() timing. \n",
540 " \"\"\"\n",
541 " \n",
542 " predictions = context.predictions \n",
543 " predictions = predictions.loc[data.can_trade(predictions.index)]\n",
544 " \n",
545 " # Select long/short positions\n",
546 " n_positions = int(min(N_POSITIONS, len(predictions)) / 2)\n",
547 " to_trade = (predictions[predictions>0]\n",
548 " .nlargest(n_positions)\n",
549 " .append(predictions[predictions < 0]\n",
550 " .nsmallest(n_positions)))\n",
551 "\n",
552 " # Model may produce duplicate predictions\n",
553 " to_trade = to_trade[~to_trade.index.duplicated()]\n",
554 " \n",
555 " # Setup Optimization Objective\n",
556 " objective = opt.MaximizeAlpha(to_trade)\n",
557 "\n",
558 " # Setup Optimization Constraints\n",
559 " constrain_gross_leverage = opt.MaxGrossExposure(1.0)\n",
560 " constrain_pos_size = opt.PositionConcentration.with_equal_bounds(-.02, .02)\n",
561 " market_neutral = opt.DollarNeutral()\n",
562 " constrain_risk = RiskModelExposure(\n",
563 " risk_model_loadings=context.risk_loading_pipeline, \n",
564 " version=opt.Newest)\n",
565 " \n",
566 " # Optimizer calculates portfolio weights and\n",
567 " # moves portfolio toward the target.\n",
568 " order_optimal_portfolio(\n",
569 " objective=objective,\n",
570 " constraints=[ \n",
571 " constrain_gross_leverage,\n",
572 " constrain_pos_size,\n",
573 " market_neutral,\n",
574 " constrain_risk\n",
575 " ],\n",
576 " )"
577 ]
578 },
579 {
580 "cell_type": "markdown",
581 "metadata": {},
582 "source": [
583 "### Track Performance"
584 ]
585 },
586 {
587 "cell_type": "code",
588 "execution_count": null,
589 "metadata": {},
590 "outputs": [],
591 "source": [
592 "def log_metrics(context, data):\n",
593 " \"\"\"\n",
594 " Plot variables at the end of each day.\n",
595 " \"\"\"\n",
596 " record(leverage=context.account.leverage,\n",
597 " #num_positions=len(context.portfolio.positions),\n",
598 " realized_rmse=context.realized_rmse,\n",
599 " realized_ic=context.realized_ic,\n",
600 " long_short_spread=context.long_short_spread,\n",
601 " )"
602 ]
603 }
604 ],
605 "metadata": {
606 "kernelspec": {
607 "display_name": "Python 3",
608 "language": "python",
609 "name": "python3"
610 },
611 "language_info": {
612 "codemirror_mode": {
613 "name": "ipython",
614 "version": 3
615 },
616 "file_extension": ".py",
617 "mimetype": "text/x-python",
618 "name": "python",
619 "nbconvert_exporter": "python",
620 "pygments_lexer": "ipython3",
621 "version": "3.6.8"
622 },
623 "toc": {
624 "base_numbering": 1,
625 "nav_menu": {},
626 "number_sections": true,
627 "sideBar": true,
628 "skip_h1_title": true,
629 "title_cell": "Table of Contents",
630 "title_sidebar": "Contents",
631 "toc_cell": false,
632 "toc_position": {},
633 "toc_section_display": true,
634 "toc_window_display": true
635 }
636 },
637 "nbformat": 4,
638 "nbformat_minor": 2
639 }