從0開始實(shí)現(xiàn)一個(gè)Adaboost分類器（完整代碼）

class?DecisionTreeClassifierWithWeight:
????def?__init__(self):
????????self.best_err?=?1??#?最小的加權(quán)錯(cuò)誤率
????????self.best_fea_id?=?0??#?最優(yōu)特征id
????????self.best_thres?=?0??#?選定特征的最優(yōu)閾值
????????self.best_op?=?1??#?閾值符號(hào)，其中?1:?>,?0:?<

????def?fit(self,?X,?y,?sample_weight=None):
????????if?sample_weight?is?None:
????????????sample_weight?=?np.ones(len(X))?/?len(X)
????????n?=?X.shape[1]
????????for?i?in?range(n):
????????????feature?=?X[:,?i]??#?選定特征列
????????????fea_unique?=?np.sort(np.unique(feature))??#?將所有特征值從小到大排序
????????????for?j?in?range(len(fea_unique)-1):
????????????????thres?=?(fea_unique[j]?+?fea_unique[j+1])?/?2??#?逐一設(shè)定可能閾值
????????????????for?op?in?(0,?1):
????????????????????y_?=?2*(feature?>=?thres)-1?if?op==1?else?2*(feature?-1??#?判斷何種符號(hào)為最優(yōu)
????????????????????err?=?np.sum((y_?!=?y)*sample_weight)
????????????????????if?err?#?當(dāng)前參數(shù)組合可以獲得更低錯(cuò)誤率，更新最優(yōu)參數(shù)
????????????????????????self.best_err?=?err
????????????????????????self.best_op?=?op
????????????????????????self.best_fea_id?=?i
????????????????????????self.best_thres?=?thres
????????return?self
????
????def?predict(self,?X):
????????feature?=?X[:,?self.best_fea_id]
????????return?2*(feature?>=?self.best_thres)-1?if?self.best_op==1?else?2*(feature?-1
????
????def?score(self,?X,?y,?sample_weight=None):
????????y_pre?=?self.predict(X)
????????if?sample_weight?is?not?None:
????????????return?np.sum((y_pre?==?y)*sample_weight)
????????return?np.mean(y_pre?==?y)

from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split

X, y = load_breast_cancer(return_X_y=True)
y = 2*y-1  # 將0/1取值映射為-1/1取值
X_train, X_test, y_train, y_test = train_test_split(X, y)

DecisionTreeClassifierWithWeight().fit(X_train, y_train).score(X_test, y_test)
# 0.8671328671328671

class AdaBoostClassifier_:
    def __init__(self, n_estimators=50):
        self.n_estimators = n_estimators
        self.estimators = []
        self.alphas = []

    def fit(self, X, y):
        sample_weight = np.ones(len(X)) / len(X)  # 初始化樣本權(quán)重為 1/N
        for _ in range(self.n_estimators):
            dtc = DecisionTreeClassifierWithWeight().fit(X, y, sample_weight)  # 訓(xùn)練弱學(xué)習(xí)器
            alpha = 1/2 * np.log((1-dtc.best_err)/dtc.best_err)  # 權(quán)重系數(shù)
            y_pred = dtc.predict(X)
            sample_weight *= np.exp(-alpha*y_pred*y)  # 更新迭代樣本權(quán)重
            sample_weight /= np.sum(sample_weight)  # 樣本權(quán)重歸一化
            self.estimators.append(dtc)
            self.alphas.append(alpha)
        return self

    def predict(self, X):
        y_pred = np.empty((len(X), self.n_estimators))  # 預(yù)測(cè)結(jié)果二維數(shù)組，其中每一列代表一個(gè)弱學(xué)習(xí)器的預(yù)測(cè)結(jié)果
        for i in range(self.n_estimators):
            y_pred[:, i] = self.estimators[i].predict(X)
        y_pred = y_pred * np.array(self.alphas)  # 將預(yù)測(cè)結(jié)果與訓(xùn)練權(quán)重乘積作為集成預(yù)測(cè)結(jié)果
        return 2*(np.sum(y_pred, axis=1)>0)-1  # 以0為閾值，判斷并映射為-1和1

    def score(self, X, y):
        y_pred = self.predict(X)
        return np.mean(y_pred==y)

from sklearn.ensemble import AdaBoostClassifier
AdaBoostClassifier_().fit(X_train, y_train).score(X_test, y_test)
# 0.986013986013986
AdaBoostClassifier().fit(X_train, y_train).score(X_test, y_test)
# 0.965034965034965