I'm using the DecisionTreeClassifier from scikit-learn to classify some data. I'm also using other algorithms and to compare them I use the area under the precision-recall metric. The problem is the shape of the AUPRC for the DecisionTreeClassifier is a square and not the usual shape you would expect for this metric.
Here is how I am calculating the AUPRC for the DecisionTreeClassifier. I had some trouble calculating this because the DecisionTreeClassifer does not have the decision_function() as do other classifiers like LogisticRegression
These are the results I got for the AUPRC of SVM, Logistic Regression, and DecisionTreeClassifier
Here is how I calculate the AUPRC for DecisionTreeClassifier
def execute(X_train, y_train, X_test, y_test):
tree = DecisionTreeClassifier(class_weight='balanced')
tree_y_score = tree.fit(X_train, y_train).predict(X_test)
tree_ap_score = average_precision_score(y_test, tree_y_score)
precision, recall, _ = precision_recall_curve(y_test, tree_y_score)
values = {'ap_score': tree_ap_score, 'precision': precision, 'recall': recall}
return values
Here is how I calculate the AUPRC for SVM:
def execute(X_train, y_train, X_test, y_test):
svm = SVC(class_weight='balanced')
svm.fit(X_train, y_train.values.ravel())
svm_y_score = svm.decision_function(X_test)
svm_ap_score = average_precision_score(y_test, svm_y_score)
precision, recall, _ = precision_recall_curve(y_test, svm_y_score)
values = {'ap_score': svm_ap_score, 'precision': precision, 'recall': recall}
return values
Here is how I calculate the AUPRC for LogisticRegression:
def execute(X_train, y_train, X_test, y_test):
lr = LogisticRegression(class_weight='balanced')
lr.fit(X_train, y_train.values.ravel())
lr_y_score = lr.decision_function(X_test)
lr_ap_score = average_precision_score(y_test, lr_y_score)
precision, recall, _ = precision_recall_curve(y_test, lr_y_score)
values = {'ap_score': lr_ap_score, 'precision': precision, 'recall': recall}
return values
I then call them methods and plot the results like this:
import LogReg_AP_Harness as lrApTest
import SVM_AP_Harness as svmApTest
import DecTree_AP_Harness as dtApTest
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import label_binarize
import matplotlib.pyplot as plt
def do_work(df):
X = df.ix[:, df.columns != 'Class']
y = df.ix[:, df.columns == 'Class']
y_binarized = label_binarize(y, classes=[0, 1])
n_classes = y_binarized.shape[1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=0)
_, _, y_train_binarized, y_test_binarized = train_test_split(X, y_binarized, test_size=.3, random_state=0)
print('Executing Logistic Regression')
lr_values = lrApTest.execute(X_train, y_train, X_test, y_test)
print('Executing Decision Tree')
dt_values = dtApTest.execute(X_train, y_train_binarized, X_test, y_test_binarized)
print('Executing SVM')
svm_values = svmApTest.execute(X_train, y_train, X_test, y_test)
plot_aupr_curves(lr_values, svm_values, dt_values)
def plot_aupr_curves(lr_values, svm_values, dt_values):
lr_ap_score = lr_values['ap_score']
lr_precision = lr_values['precision']
lr_recall = lr_values['recall']
svm_ap_score = svm_values['ap_score']
svm_precision = svm_values['precision']
svm_recall = svm_values['recall']
dt_ap_score = dt_values['ap_score']
dt_precision = dt_values['precision']
dt_recall = dt_values['recall']
plt.step(svm_recall, svm_precision, color='g', alpha=0.2,where='post')
plt.fill_between(svm_recall, svm_precision, step='post', alpha=0.2, color='g')
plt.step(lr_recall, lr_precision, color='b', alpha=0.2, where='post')
plt.fill_between(lr_recall, lr_precision, step='post', alpha=0.2, color='b')
plt.step(dt_recall, dt_precision, color='r', alpha=0.2, where='post')
plt.fill_between(dt_recall, dt_precision, step='post', alpha=0.2, color='r')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('SVM (Green): Precision-Recall curve: AP={0:0.2f}'.format(svm_ap_score) + '\n' +
'Logistic Regression (Blue): Precision-Recall curve: AP={0:0.2f}'.format(lr_ap_score) + '\n' +
'Decision Tree (Red): Precision-Recall curve: AP={0:0.2f}'.format(dt_ap_score))
plt.show()
In the the do_work() method I had to binarize y because DecisionTreeClassifier does not have a descision_function(). I had the approach from here.
This is the plot:
I guess what it boils down to is that I'm calculating the AUPRC for DecisionTreeClassifier incorrectly.
