Skip to content

GitLab

Commit 3959271c authored by Billy Amélie's avatar Billy Amélie
Browse files

Pull master sur AmelieBranch

parent 10c6a860
Hide whitespace changes
Inline Side-by-side
old_Amelie/CNN.ipynb 0 → 100644
View file @ 3959271c
This source diff could not be displayed because it is too large. You can view the blob instead.
old_Amelie/Regression.ipynb 0 → 100644
View file @ 3959271c
%% Cell type:code id: tags:
``` python
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import matplotlib.pyplot as plt
import os
from pathlib import Path
from sklearn.model_selection import KFold
from sklearn.metrics import mean_absolute_error
```
%% Cell type:code id: tags:
``` python
input_directory='../osic-pulmonary-fibrosis-progression'
```
%% Cell type:code id: tags:
``` python
# Reading data
test_df = pd.read_csv(f'{input_directory}/test.csv')
sample_sub= pd.read_csv(f'{input_directory}/sample_submission.csv')
train_df= pd.read_csv(f'{input_directory}/train.csv')
train_df.head()
```
%%%% Output: execute_result
Patient Weeks FVC Percent Age Sex SmokingStatus
0 ID00007637202177411956430 -4 2315 58.253649 79 Male Ex-smoker
1 ID00007637202177411956430 5 2214 55.712129 79 Male Ex-smoker
2 ID00007637202177411956430 7 2061 51.862104 79 Male Ex-smoker
3 ID00007637202177411956430 9 2144 53.950679 79 Male Ex-smoker
4 ID00007637202177411956430 11 2069 52.063412 79 Male Ex-smoker
%% Cell type:code id: tags:
``` python
patients_train_ids= train_df.Patient.unique()
patient_test_list= test_df.Patient.unique()
list_p = train_df.Patient.isin( patient_test_list )
patients_train_ids = [pat for pat in patients_train_ids if not pat in list_p]
```
%% Cell type:code id: tags:
``` python
le = LabelEncoder()
le.fit(train_df['Sex'])
print(list(le.classes_))
train_df['Sex']=le.transform(train_df['Sex'])
test_df['Sex']=le.transform(test_df['Sex'])
```
%%%% Output: stream
['Female', 'Male']
%% Cell type:code id: tags:
``` python
le = LabelEncoder()
le.fit(train_df['SmokingStatus'])
print(list(le.classes_))
train_df['SmokingStatus']=le.transform(train_df['SmokingStatus'])
test_df['SmokingStatus']=le.transform(test_df['SmokingStatus'])
```
%%%% Output: stream
['Currently smokes', 'Ex-smoker', 'Never smoked']
%% Cell type:code id: tags:
``` python
# new dataframe with one row per patient for training
train_data = []
for patient in patients_train_ids :
#select all data related to a patient
patientData = train_df[train_df['Patient'] == patient]
# save first measurements
firstMeasure = list(patientData.iloc[0, :].values)
#for ech measurement, add fist measurement and duration since first measurement
for i, week in enumerate(patientData['Weeks'].iloc[1:]):
fvc, fvc_pctg = patientData.iloc[i, 2], patientData.iloc[i, 3]
trainDataPoint = firstMeasure + [week, fvc]
train_data.append(trainDataPoint)
training_df = pd.DataFrame(train_data)
training_df.columns = ['PatientID', 'First_week', 'First_FVC', 'First_Percent', 'Age', 'Sex', 'SmokingStatus'] + ['target_week', 'Target_FVC']
training_df['Delta_week'] = training_df['target_week'] - training_df['First_week']
#rearrange columns
training_df = training_df[['PatientID','Age','Sex','SmokingStatus', 'First_FVC', 'First_Percent','Delta_week','Target_FVC']]
```
%% Cell type:code id: tags:
``` python
training_df
```
%%%% Output: execute_result
PatientID Age Sex SmokingStatus First_FVC \
0 ID00007637202177411956430 79 1 1 2315
1 ID00007637202177411956430 79 1 1 2315
2 ID00007637202177411956430 79 1 1 2315
3 ID00007637202177411956430 79 1 1 2315
4 ID00007637202177411956430 79 1 1 2315
... ... ... ... ... ...
1368 ID00426637202313170790466 73 1 2 2925
1369 ID00426637202313170790466 73 1 2 2925
1370 ID00426637202313170790466 73 1 2 2925
1371 ID00426637202313170790466 73 1 2 2925
1372 ID00426637202313170790466 73 1 2 2925
First_Percent Delta_week Target_FVC
0 58.253649 9 2315
1 58.253649 11 2214
2 58.253649 13 2061
3 58.253649 15 2144
4 58.253649 21 2069
... ... ... ...
1368 71.824968 13 2976
1369 71.824968 19 2712
1370 71.824968 31 2978
1371 71.824968 43 2908
1372 71.824968 59 2975
[1373 rows x 8 columns]
%% Cell type:code id: tags:
``` python
sample_sub.head(-20) #How the result must look like
```
%%%% Output: execute_result
Patient_Week FVC Confidence
0 ID00419637202311204720264_-12 2000 100
1 ID00421637202311550012437_-12 2000 100
2 ID00422637202311677017371_-12 2000 100
3 ID00423637202312137826377_-12 2000 100
4 ID00426637202313170790466_-12 2000 100
.. ... ... ...
705 ID00419637202311204720264_129 2000 100
706 ID00421637202311550012437_129 2000 100
707 ID00422637202311677017371_129 2000 100
708 ID00423637202312137826377_129 2000 100
709 ID00426637202313170790466_129 2000 100
[710 rows x 3 columns]
%% Cell type:code id: tags:
``` python
sample_sub[['Patient','Week']] = sample_sub['Patient_Week'].str.split('_',expand=True)
```
%% Cell type:code id: tags:
``` python
sample_sub.head(-3)
```
%%%% Output: execute_result
Patient_Week FVC Confidence \
0 ID00419637202311204720264_-12 2000 100
1 ID00421637202311550012437_-12 2000 100
2 ID00422637202311677017371_-12 2000 100
3 ID00423637202312137826377_-12 2000 100
4 ID00426637202313170790466_-12 2000 100
.. ... ... ...
722 ID00422637202311677017371_132 2000 100
723 ID00423637202312137826377_132 2000 100
724 ID00426637202313170790466_132 2000 100
725 ID00419637202311204720264_133 2000 100
726 ID00421637202311550012437_133 2000 100
Patient Week
0 ID00419637202311204720264 -12
1 ID00421637202311550012437 -12
2 ID00422637202311677017371 -12
3 ID00423637202312137826377 -12
4 ID00426637202313170790466 -12
.. ... ...
722 ID00422637202311677017371 132
723 ID00423637202312137826377 132
724 ID00426637202313170790466 132
725 ID00419637202311204720264 133
726 ID00421637202311550012437 133
[727 rows x 5 columns]
%% Cell type:code id: tags:
``` python
test_df
```
%%%% Output: execute_result
Patient Weeks FVC Percent Age Sex SmokingStatus
0 ID00419637202311204720264 6 3020 70.186855 73 1 1
1 ID00421637202311550012437 15 2739 82.045291 68 1 1
2 ID00422637202311677017371 6 1930 76.672493 73 1 1
3 ID00423637202312137826377 17 3294 79.258903 72 1 1
4 ID00426637202313170790466 0 2925 71.824968 73 1 2
%% Cell type:code id: tags:
``` python
p= patient_test_list[0]
patientData = test_df[test_df['Patient'] == p]
firstMeasure = list(patientData.iloc[0, :].values)
s_patient = sample_sub[sample_sub['Patient']==p]
```
%% Cell type:markdown id: tags:
We will preduct, for each patient the test set, their FVC for the weeks that are in the sample set for them (this is a requirement from the competition):
"For each Patient_Week from the submission file, you must predict the FVC and a confidence"
%% Cell type:code id: tags:
``` python
# create testing data
testData = []
for p in patient_test_list:
patientData = test_df[test_df['Patient'] == p]
firstMeasure = list(patientData.iloc[0, :].values)
s_patient = sample_sub[sample_sub['Patient']==p]
allweeks= s_patient['Week'].tolist()
for week in allweeks:
testDataPoint = firstMeasure + [week]
testData.append(testDataPoint)
testData = pd.DataFrame(testData)
testData.columns = ['PatientID', 'first_week', 'First_FVC', 'First_Percent', 'Age', 'Sex', 'SmokingStatus'] + ['Target_week']
testData['Delta_week'] = testData['Target_week'].map(int) - testData['first_week']
#Rearange
testData = testData[['PatientID','Age','Sex','SmokingStatus', 'First_FVC', 'First_Percent','Delta_week','Target_week']]
```
%% Cell type:code id: tags:
``` python
testData.head(3)
```
%%%% Output: execute_result
PatientID Age Sex SmokingStatus First_FVC \
0 ID00419637202311204720264 73 1 1 3020
1 ID00419637202311204720264 73 1 1 3020
2 ID00419637202311204720264 73 1 1 3020
First_Percent Delta_week Target_week
0 70.186855 -18 -12
1 70.186855 -17 -11
2 70.186855 -16 -10
%% Cell type:markdown id: tags:
# Model
%% Cell type:code id: tags:
``` python
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
model = LinearRegression()
model.fit(training_df.drop(columns = ['PatientID', 'Target_FVC']), training_df['Target_FVC'])
print('Training accuracy:',model.score(training_df.drop(columns = ['PatientID', 'Target_FVC']), training_df['Target_FVC']))
prediction_test = model.predict(testData.drop(columns = ['PatientID','Target_week']))
prediction_train =model.predict(training_df.drop(columns = ['PatientID', 'Target_FVC']))
```
%%%% Output: stream
Training accuracy: 0.9414634426681496
%% Cell type:markdown id: tags:
# Evaluation
%% Cell type:markdown id: tags:
This competition is evaluated on a modified version of the Laplace Log Likelihood. In medical applications, it is useful to evaluate a model's confidence in its decisions. Accordingly, the metric is designed to reflect both the accuracy and certainty of each prediction.For each true FVC measurement, you will predict both an FVC and a confidence measure (standard deviation $\sigma$). The metric is computed as:
$\sigma_{clipped} = max(\sigma, 70)$,
$\Delta = min ( |FVC_{true} - FVC_{predicted}|, 1000 )$
$metric = - \frac{\sqrt{2} \Delta}{\sigma_{clipped}} - \ln ( \sqrt{2} \sigma_{clipped} )$
The final score is calculated by averaging the metric across all test set Patient_Weeks (three per patient). Note that metric values will be negative and higher is better.
%% Cell type:code id: tags:
``` python
import math
def score(y_true, y_pred):
sigma = ( y_true - y_pred ) #########
fvc_pred = y_pred
sigma_clip = np.maximum(sigma, 70)
delta = np.minimum(abs(y_true - fvc_pred),1000)
sq2 = math.sqrt(2)
metric = -(delta / sigma_clip)*sq2 - np.log(sigma_clip* sq2)
return (sigma, np.mean(metric))
```
%% Cell type:code id: tags:
``` python
conf,score = score(prediction_train,training_df['Target_FVC'])
print(score)
```
%%%% Output: stream
-6.730931933436616
%% Cell type:code id: tags:
``` python
sub = []
for i in range(testData.shape[0]):
patient, week, pred = testData.loc[i, 'PatientID'], testData.loc[i, 'Target_week'], prediction_test[i]
confidence = 0 #TBD
sub.append([patient + '_' + str(week), pred, confidence])
sub = pd.DataFrame(sub)
sub.columns = ['Patient_Week', 'FVC', 'Confidence']
```
%% Cell type:code id: tags:
``` python
```
%% Cell type:code id: tags:
``` python
patient_test_list = patient_test_list.tolist()
```
%% Cell type:code id: tags:
``` python
train_df_ini= pd.read_csv(f'{input_directory}/train.csv')
train_df_ini[train_df_ini.Patient.isin(patient_test_list)]
```
%%%% Output: execute_result
Patient Weeks FVC Percent Age Sex \
1504 ID00419637202311204720264 6 3020 70.186855 73 Male
1505 ID00419637202311204720264 7 2859 66.445106 73 Male
1506 ID00419637202311204720264 9 2783 64.678814 73 Male
1507 ID00419637202311204720264 10 2719 63.191410 73 Male
1508 ID00419637202311204720264 13 2738 63.632983 73 Male
1509 ID00419637202311204720264 18 2694 62.610393 73 Male
1510 ID00419637202311204720264 31 2708 62.935763 73 Male
1511 ID00419637202311204720264 43 2793 64.911221 73 Male
1512 ID00419637202311204720264 59 2727 63.377336 73 Male
1513 ID00421637202311550012437 15 2739 82.045291 68 Male
1514 ID00421637202311550012437 17 2756 82.554517 68 Male
1515 ID00421637202311550012437 19 2755 82.524563 68 Male
1516 ID00421637202311550012437 21 2820 84.471603 68 Male
1517 ID00421637202311550012437 23 2853 85.460101 68 Male
1518 ID00421637202311550012437 29 2716 81.356338 68 Male
1519 ID00421637202311550012437 41 2833 84.861011 68 Male
1520 ID00421637202311550012437 54 2771 83.003834 68 Male
1521 ID00421637202311550012437 70 2628 78.720345 68 Male
1522 ID00421637202311550012437 70 2719 81.446202 68 Male
1523 ID00422637202311677017371 6 1930 76.672493 73 Male
1524 ID00422637202311677017371 11 1936 76.910853 73 Male
1525 ID00422637202311677017371 13 1955 77.665660 73 Male
1526 ID00422637202311677017371 15 1848 73.414905 73 Male
1527 ID00422637202311677017371 17 1897 75.361513 73 Male
1528 ID00422637202311677017371 23 1946 77.308120 73 Male
1529 ID00422637202311677017371 35 1862 73.971079 73 Male
1530 ID00422637202311677017371 47 1713 68.051804 73 Male
1531 ID00423637202312137826377 17 3294 79.258903 72 Male
1532 ID00423637202312137826377 18 2777 66.819057 72 Male
1533 ID00423637202312137826377 19 2700 64.966314 72 Male
1534 ID00423637202312137826377 21 3014 72.521655 72 Male
1535 ID00423637202312137826377 23 2661 64.027911 72 Male
1536 ID00423637202312137826377 30 2778 66.843118 72 Male
1537 ID00423637202312137826377 42 2516 60.538980 72 Male
1538 ID00423637202312137826377 53 2432 58.517806 72 Male
1539 ID00423637202312137826377 70 2578 62.030799 72 Male
1540 ID00426637202313170790466 0 2925 71.824968 73 Male
1541 ID00426637202313170790466 7 2903 71.284746 73 Male
1542 ID00426637202313170790466 9 2916 71.603968 73 Male
1543 ID00426637202313170790466 11 2976 73.077301 73 Male
1544 ID00426637202313170790466 13 2712 66.594637 73 Male
1545 ID00426637202313170790466 19 2978 73.126412 73 Male
1546 ID00426637202313170790466 31 2908 71.407524 73 Male
1547 ID00426637202313170790466 43 2975 73.052745 73 Male
1548 ID00426637202313170790466 59 2774 68.117081 73 Male
SmokingStatus
1504 Ex-smoker
1505 Ex-smoker
1506 Ex-smoker
1507 Ex-smoker
1508 Ex-smoker
1509 Ex-smoker
1510 Ex-smoker
1511 Ex-smoker
1512 Ex-smoker
1513 Ex-smoker
1514 Ex-smoker
1515 Ex-smoker
1516 Ex-smoker
1517 Ex-smoker
1518 Ex-smoker
1519 Ex-smoker
1520 Ex-smoker
1521 Ex-smoker
1522 Ex-smoker
1523 Ex-smoker
1524 Ex-smoker
1525 Ex-smoker
1526 Ex-smoker
1527 Ex-smoker
1528 Ex-smoker
1529 Ex-smoker
1530 Ex-smoker
1531 Ex-smoker
1532 Ex-smoker
1533 Ex-smoker
1534 Ex-smoker
1535 Ex-smoker
1536 Ex-smoker
1537 Ex-smoker
1538 Ex-smoker
1539 Ex-smoker
1540 Never smoked
1541 Never smoked
1542 Never smoked
1543 Never smoked
1544 Never smoked
1545 Never smoked
1546 Never smoked
1547 Never smoked
1548 Never smoked
%% Cell type:code id: tags:
``` python
df1 = sub.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True)
```
%% Cell type:code id: tags:
``` python
df1
```
%%%% Output: execute_result
Patient_Week FVC Confidence
0 ID00419637202311204720264_-1 2998.380595 0
1 ID00419637202311204720264_-10 3022.589358 0
2 ID00419637202311204720264_-11 3025.279220 0
3 ID00419637202311204720264_-12 3027.969083 0
4 ID00419637202311204720264_-2 3001.070458 0
.. ... ... ...
725 ID00426637202313170790466_95 2624.509718 0
726 ID00426637202313170790466_96 2621.819855 0
727 ID00426637202313170790466_97 2619.129993 0
728 ID00426637202313170790466_98 2616.440130 0
729 ID00426637202313170790466_99 2613.750268 0
[730 rows x 3 columns]
%% Cell type:code id: tags:
``` python
```
old_Amelie/effnet_3img.ipynb 0 → 100644
View file @ 3959271c
This source diff could not be displayed because it is too large. You can view the blob instead.
old_Amelie/k-fold-efficientnet-models-training.ipynb 0 → 100644
View file @ 3959271c
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"papermill": {
"duration": 0.012261,
"end_time": "2020-08-20T13:10:33.841122",
"exception": false,
"start_time": "2020-08-20T13:10:33.828861",
"status": "completed"
},
"tags": []
},
"source": [
"# Overview & Remarks\n",
"\n",
"This notebook contains the configurations required to train an efficientnet model for K-folds.\n",
"\n",
"It is possible to hit -0.6910 LB by tweaking parameters in this notebook!"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Collecting opencv-python\n",
" Downloading opencv_python-4.5.1.48-cp37-cp37m-win_amd64.whl (34.9 MB)\n",
"Requirement already satisfied: numpy>=1.14.5 in c:\\users\\am\\anaconda3\\lib\\site-packages (from opencv-python) (1.19.5)\n",
"Installing collected packages: opencv-python\n",
"Successfully installed opencv-python-4.5.1.48\n"
]
}
],
"source": [
"!pip install opencv-python"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
"_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
"papermill": {
"duration": 6.82276,
"end_time": "2020-08-20T13:11:01.20963",
"exception": false,
"start_time": "2020-08-20T13:10:54.38687",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"import os\n",
"import cv2\n",
"import pydicom\n",
"import pandas as pd\n",
"import numpy as np \n",
"import tensorflow as tf \n",
"import matplotlib.pyplot as plt \n",
"from tqdm.notebook import tqdm \n",
"from tensorflow.keras.layers import (\n",
" Dense, Dropout, Activation, Flatten, Input, BatchNormalization, GlobalAveragePooling2D, Add, Conv2D, AveragePooling2D, \n",
" LeakyReLU, Concatenate \n",
")\n",
"from tensorflow.keras import Model\n",
"from tensorflow.keras.utils import Sequence\n",
"import tensorflow.keras.backend as K\n",
"import tensorflow.keras.applications as tfa\n",
"import efficientnet.tfkeras as efn\n",
"from sklearn.model_selection import train_test_split, KFold\n",
"import seaborn as sns"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"papermill": {
"duration": 2.514265,
"end_time": "2020-08-20T13:11:03.73512",
"exception": false,
"start_time": "2020-08-20T13:11:01.220855",
"status": "completed"
},
"tags": []
},
"outputs": [],
"source": [
"config = tf.compat.v1.ConfigProto()\n",
"config.gpu_options.allow_growth = True\n",
"session = tf.compat.v1.Session(config=config)"
]
},
{
"cell_type": "markdown",
"metadata": {
"papermill": {
"duration": 0.008417,
"end_time": "2020-08-20T13:11:03.751195",
"exception": false,
"start_time": "2020-08-20T13:11:03.742778",
"status": "completed"
},
"tags": []