量化投資的強(qiáng)化學(xué)習(xí)神器！FinRL 入門(mén)指南

# 公眾號(hào)：二七阿爾量化
from finrl import config
from finrl import config_tickers
from finrl.main import check_and_make_directories

import pandas as pd

from finrl.finrl_meta.preprocessor.yahoodownloader import YahooDownloader
from finrl.finrl_meta.preprocessor.preprocessors import FeatureEngineer, data_split
from finrl.finrl_meta.env_stock_trading.env_stocktrading import StockTradingEnv
from finrl.agents.stablebaselines3.models import DRLAgent

import sys
sys.path.append("../FinRL-Library")

import itertools

from finrl.config import (
    DATA_SAVE_DIR,
    TRAINED_MODEL_DIR,
    TENSORBOARD_LOG_DIR,
    RESULTS_DIR,
)

check_and_make_directories([DATA_SAVE_DIR, TRAINED_MODEL_DIR, TENSORBOARD_LOG_DIR, RESULTS_DIR])


'''
# Part 1. 下載數(shù)據(jù)
'''

df = YahooDownloader(
    start_date='2009-01-01',
    end_date='2021-10-31',
    ticker_list=config_tickers.DOW_30_TICKER
).fetch_data(proxy={"http": "http://127.0.0.1:10809", "https": "https://127.0.0.1:10809"})


print(f"config_tickers.DOW_30_TICKER: {config_tickers.DOW_30_TICKER}")


print(f"df.shape: {df.shape}")


df.sort_values(['date','tic'],ignore_index=True).head()


'''
# Part 2: 數(shù)據(jù)預(yù)處理
'''

fe = FeatureEngineer(
                    use_technical_indicator=True,
                    tech_indicator_list=config.INDICATORS,
                    use_vix=True,
                    use_turbulence=True,
                    user_defined_feature = False)

processed = fe.preprocess_data(df)


list_ticker = processed["tic"].unique().tolist()
list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))
combination = list(itertools.product(list_date,list_ticker))

processed_full = pd.DataFrame(combination,columns=["date","tic"]).merge(processed,on=["date","tic"],how="left")
processed_full = processed_full[processed_full['date'].isin(processed['date'])]
processed_full = processed_full.sort_values(['date','tic'])

processed_full = processed_full.fillna(0)


processed_full.sort_values(['date','tic'],ignore_index=True).head(10)

# 訓(xùn)練集
train = data_split(processed_full, '2009-01-01','2020-07-01')
# 測(cè)試集
trade = data_split(processed_full, '2020-07-01','2021-10-31')

print(f"len(train): {len(train)}")
print(f"len(trade): {len(trade)}")
print(f"train.tail(): {train.tail()}")
print(f"trade.head(): {trade.head()}")
print(f"config.INDICATORS: {config.INDICATORS}")
stock_dimension = len(train.tic.unique())
state_space = 1 + 2*stock_dimension + len(config.INDICATORS)*stock_dimension
print(f"Stock Dimension: {stock_dimension}, State Space: {state_space}")

buy_cost_list = sell_cost_list = [0.001] * stock_dimension
num_stock_shares = [0] * stock_dimension

env_kwargs = {
    "hmax": 100,
    "initial_amount": 1000000,
    "num_stock_shares": num_stock_shares,
    "buy_cost_pct": buy_cost_list,
    "sell_cost_pct": sell_cost_list,
    "state_space": state_space,
    "stock_dim": stock_dimension,
    "tech_indicator_list": config.INDICATORS,
    "action_space": stock_dimension,
    "reward_scaling": 1e-4
}


e_train_gym = StockTradingEnv(df = train, **env_kwargs)

env_train, _ = e_train_gym.get_sb_env()
print(f"type(env_train): {type(env_train)}")


'''
# Part 3: 模型訓(xùn)練
'''
agent = DRLAgent(env = env_train)
SAC_PARAMS = {
    "batch_size": 128,
    "buffer_size": 1000000,
    "learning_rate": 0.0001,
    "learning_starts": 100,
    "ent_coef": "auto_0.1",
}

model_sac = agent.get_model("sac", model_kwargs = SAC_PARAMS)


trained_sac = agent.train_model(model=model_sac,
                             tb_log_name='sac',
                             total_timesteps=60000)