CS634-Project/mlp-binary.py at main · Pendon13/CS634-Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve, auc, roc_auc_score, classification_report
from sklearn.metrics import RocCurveDisplay
import pandas as pd
import numpy as np
import joblib
import matplotlib.pyplot as plt
import time
from pathlib import Path
Path("./model/binary").mkdir(parents=True, exist_ok=True)
#hidden_layer = (8, 3, 8)
#hidden_layer = (1963,)
hidden_layer = (841, 241, 69, 20, 6, 2)

# We change the following to binary
# No radiation and/or cancer-directed surgery is 0
# Some form of surgery is 1

# y_enc_classes = ["No radiation and/or cancer-directed surgery",
#                 "Intraoperative rad with other rad before/after surgery",
#                 "Intraoperative radiation",
#                 "Radiation after surgery",
#                 "Radiation before and after surgery",
#                 "Radiation prior to surgery",
#                 "Sequence unknown, but both were given",
#                 "Surgery both before and after radiation"
#                 ]

y_enc_classes = ["No radiation and/or cancer-directed surgery",
                 "Surgery given"]

print("loading...")

X = pd.read_csv("2020-extra_removed_preprocessed.csv", header=0, dtype=int)
y = pd.read_csv("2020-extra_removed_preprocessed_target_binary.csv", header=0, dtype=int)
y = y[["0"]].values.ravel()


# Assuming 'X' is your feature matrix and 'y' is your target variable

# Split your data into training and testing sets
print("split")
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, test_size=0.2, random_state=42)
print("model start")
# Create and train the MLPClassifier
n_iter_max = 100000
start_time = time.time()
clf = MLPClassifier(hidden_layer_sizes=hidden_layer,
                    max_iter=n_iter_max,
                    alpha = 0.0001,
                    activation='relu',
                    solver='adam',
                    random_state=42,
                    tol=1e-15)
try:
    print('try loading')
    clf = joblib.load('./model/binary/mlp-'+ str(hidden_layer) + '-main.joblib')
    timeTaken = 0
    print('load success')
except:
    print('fitting')
    with joblib.parallel_backend('threading', n_jobs=2):
        clf.fit(X_train, y_train)
    timeTaken = time.time()-start_time
    print(timeTaken)
    print('fit done')
    try:
        print('saving clf')
        joblib.dump(clf, './model/binary/mlp-'+ str(hidden_layer) + '-main.joblib')
    except:
        print('failed to joblib dump')


y_pred = clf.predict(X_test)
y_pred_proba = clf.predict_proba(X_test)[:, 1]
clf_params = clf.get_params()
clf_score = clf.score(X_test, y_test)
y_score = y_pred_proba

f = open("./model/binary/clf-mlp-y_pred" + str(hidden_layer) + ".txt",'a')
for i in y_pred:
    f.write(str(i) + '\n')
f.close()
f = open("./model/binary/clf-mlp-y_test" + str(hidden_layer) + ".txt",'a')
for i in y_test:
    f.write(str(i) + '\n')
f.close()

f = open("./model/binary/clf-mlp-" + str(hidden_layer) + ".txt",'a')
f.write(str(clf_params)+"\n")
f.write("predict_probab" + str(y_pred_proba) +"\n")
f.write(classification_report(y_test, y_pred))
f.write('\nThe number of iterations was: ' + str(clf.n_iter_))  # the number of iterations
f.write('\nThe final loss is ' + str(clf.loss_))            # total loss
f.write('\nThe output activation is ' + str(clf.out_activation_))
f.write("\nscore: "+ str(clf_score)+"\n")
f.write("time: " + str(timeTaken))
f.write("#"*50+"\n")
f.close()

from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix

cm = confusion_matrix(y_test, y_pred)

cm_display = ConfusionMatrixDisplay(cm).plot()
cm_display.figure_.savefig('./model/binary/mlp-'+ str(hidden_layer) + '-main.png')

from sklearn.metrics import RocCurveDisplay, roc_curve

fpr, tpr, _ = roc_curve(y_test, y_score, pos_label=clf.classes_[1])
roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot()
roc_display.figure_.savefig('./model/binary/mlp-'+ str(hidden_layer) + '-main.png')

from sklearn.metrics import PrecisionRecallDisplay, precision_recall_curve

prec, recall, _ = precision_recall_curve(y_test, y_score, pos_label=clf.classes_[1])
pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot()
pr_display.figure_.savefig('./model/binary/mlp-'+ str(hidden_layer) + '-main.png')