In [211]:
# Logistic Regression + Genetic Algorithm
# Dataset => https://archive.ics.uci.edu/ml/datasets/Autism+Screening+Adult
from scipy.io import arff
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sb
In [212]:
data = arff.loadarff('Autism-Adult-Data.arff')

## 'data' is an array with 2 elements in it.
## First being the data itself and Second being the description of data
## We load the data into a dataframe object
df = pd.DataFrame(data[0])
In [213]:
## Checking Dimension of data
df.shape
Out[213]:
(704, 21)
In [214]:
## Print the first 5 rows of the dataset
df.head()
Out[214]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 b'1' b'1' b'1' b'1' b'0' b'0' b'1' b'1' b'0' b'0' ... b'f' b'White-European' b'no' b'no' b'United States' b'no' 6.0 b'18 and more' b'Self' b'NO'
1 b'1' b'1' b'0' b'1' b'0' b'0' b'0' b'1' b'0' b'1' ... b'm' b'Latino' b'no' b'yes' b'Brazil' b'no' 5.0 b'18 and more' b'Self' b'NO'
2 b'1' b'1' b'0' b'1' b'1' b'0' b'1' b'1' b'1' b'1' ... b'm' b'Latino' b'yes' b'yes' b'Spain' b'no' 8.0 b'18 and more' b'Parent' b'YES'
3 b'1' b'1' b'0' b'1' b'0' b'0' b'1' b'1' b'0' b'1' ... b'f' b'White-European' b'no' b'yes' b'United States' b'no' 6.0 b'18 and more' b'Self' b'NO'
4 b'1' b'0' b'0' b'0' b'0' b'0' b'0' b'1' b'0' b'0' ... b'f' b'?' b'no' b'no' b'Egypt' b'no' 2.0 b'18 and more' b'?' b'NO'

5 rows × 21 columns

In [215]:
#List and store all the columns
Columns = df.columns
Columns_count = df.columns.nunique()

print(Columns)
print('Number of columns :',Columns_count)
Index(['A1_Score', 'A2_Score', 'A3_Score', 'A4_Score', 'A5_Score', 'A6_Score',
       'A7_Score', 'A8_Score', 'A9_Score', 'A10_Score', 'age', 'gender',
       'ethnicity', 'jundice', 'austim', 'contry_of_res', 'used_app_before',
       'result', 'age_desc', 'relation', 'Class/ASD'],
      dtype='object')
Number of columns : 21
In [216]:
# Check datatype of each column
df.dtypes
Out[216]:
A1_Score            object
A2_Score            object
A3_Score            object
A4_Score            object
A5_Score            object
A6_Score            object
A7_Score            object
A8_Score            object
A9_Score            object
A10_Score           object
age                float64
gender              object
ethnicity           object
jundice             object
austim              object
contry_of_res       object
used_app_before     object
result             float64
age_desc            object
relation            object
Class/ASD           object
dtype: object
In [217]:
# DATA CLEANING =>
In [218]:
# 1. Removing the b'' from each column
df2 = df.copy()

for ind, val in df.iterrows():
    for i in Columns:
        req = val[i]
        if str(req)[0] == 'b':
            split_data = str(req).split("'")
            if split_data[1].isdigit():
                actual_data = int(split_data[1])
            else:
                actual_data = split_data[1]
        df2[i][ind] = actual_data
df2.head()
<ipython-input-218-1fc535102c72>:13: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  df2[i][ind] = actual_data
c:\users\lenovo\appdata\local\programs\python\python38\lib\site-packages\pandas\core\indexing.py:671: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  self._setitem_with_indexer(indexer, value)
Out[218]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... f White-European no no United States no no 18 and more Self NO
1 1 1 0 1 0 0 0 1 0 1 ... m Latino no yes Brazil no no 18 and more Self NO
2 1 1 0 1 1 0 1 1 1 1 ... m Latino yes yes Spain no no 18 and more Parent YES
3 1 1 0 1 0 0 1 1 0 1 ... f White-European no yes United States no no 18 and more Self NO
4 1 0 0 0 0 0 0 1 0 0 ... f ? no no Egypt no no 18 and more ? NO

5 rows × 21 columns

In [219]:
# 2. Handling missing values
Col_with_questionmark = set()
counter = 0
for ind, val in df2.iterrows():
    for i in Columns:
        req = val[i]
        if req == '?':
            Col_with_questionmark.add(i)
            counter += 1
            
print(Col_with_questionmark)
print(counter)
{'ethnicity', 'relation'}
190
In [220]:
df2 = df2[~df2['relation'].isin(['?'])]
df2.head()
Out[220]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... f White-European no no United States no no 18 and more Self NO
1 1 1 0 1 0 0 0 1 0 1 ... m Latino no yes Brazil no no 18 and more Self NO
2 1 1 0 1 1 0 1 1 1 1 ... m Latino yes yes Spain no no 18 and more Parent YES
3 1 1 0 1 0 0 1 1 0 1 ... f White-European no yes United States no no 18 and more Self NO
5 1 1 1 1 1 0 1 1 1 1 ... m Others yes no United States no no 18 and more Self YES

5 rows × 21 columns

In [221]:
# 3. Assigning proper types
df2['A1_Score'] = df2['A1_Score'].astype(str).astype(int)
df2['A2_Score'] = df2['A2_Score'].astype(str).astype(int)
df2['A3_Score'] = df2['A3_Score'].astype(str).astype(int)
df2['A4_Score'] = df2['A4_Score'].astype(str).astype(int)
df2['A5_Score'] = df2['A5_Score'].astype(str).astype(int)
df2['A6_Score'] = df2['A6_Score'].astype(str).astype(int)
df2['A7_Score'] = df2['A7_Score'].astype(str).astype(int)
df2['A8_Score'] = df2['A8_Score'].astype(str).astype(int)
df2['A9_Score'] = df2['A9_Score'].astype(str).astype(int)
df2['A10_Score'] = df2['A10_Score'].astype(str).astype(int)
df2['age'] = df2['age'].astype(str).astype(float)
df2['gender'] = df2['gender'].astype(str)
df2['ethnicity'] = df2['ethnicity'].astype(str)
df2['jundice'] = df2['jundice'].astype(str)
df2['austim'] = df2['austim'].astype(str)
df2['contry_of_res'] = df2['contry_of_res'].astype(str)
df2['used_app_before'] = df2['used_app_before'].astype(str)
df2['result'] = df2['result'].astype(str)
df2['age_desc'] = df2['age_desc'].astype(str)
df2['relation'] = df2['relation'].astype(str)
df2['Class/ASD'] = df2['Class/ASD'].astype(str)
In [222]:
df2.dtypes
Out[222]:
A1_Score             int32
A2_Score             int32
A3_Score             int32
A4_Score             int32
A5_Score             int32
A6_Score             int32
A7_Score             int32
A8_Score             int32
A9_Score             int32
A10_Score            int32
age                float64
gender              object
ethnicity           object
jundice             object
austim              object
contry_of_res       object
used_app_before     object
result              object
age_desc            object
relation            object
Class/ASD           object
dtype: object
In [223]:
# Converting the string labels of 'Class/ASD' column which is our target to int
df2['Class/ASD'] = df2['Class/ASD'].map({'YES':1, 'NO':0})
df2.head()
Out[223]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... f White-European no no United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... m Latino no yes Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... m Latino yes yes Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... f White-European no yes United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... m Others yes no United States no no 18 and more Self 1

5 rows × 21 columns

In [224]:
# Converting the string labels of 'gender' column to int
df2['gender'] = df2['gender'].map({'f':1, 'm':0})
df2.head()
Out[224]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... 1 White-European no no United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... 0 Latino no yes Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... 0 Latino yes yes Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... 1 White-European no yes United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... 0 Others yes no United States no no 18 and more Self 1

5 rows × 21 columns

In [225]:
# Assigning nmerical values to 'ethnicity' column string values
df2['ethnicity'] = pd.factorize(df2['ethnicity'])[0]
df2.head()
Out[225]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... 1 0 no no United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... 0 1 no yes Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... 0 1 yes yes Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... 1 0 no yes United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... 0 2 yes no United States no no 18 and more Self 1

5 rows × 21 columns

In [226]:
# Converting the string labels of 'austim' column to int
df2['austim'] = df2['austim'].map({'yes':1, 'no':0})
df2.head()
Out[226]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... 1 0 no 0 United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... 0 1 no 1 Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... 0 1 yes 1 Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... 1 0 no 1 United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... 0 2 yes 0 United States no no 18 and more Self 1

5 rows × 21 columns

In [227]:
# Converting the string labels of 'jundice' column to int
df2['jundice'] = df2['jundice'].map({'yes':1, 'no':0})
df2.head()
Out[227]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... 1 0 0 0 United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... 0 1 0 1 Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... 0 1 1 1 Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... 1 0 0 1 United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... 0 2 1 0 United States no no 18 and more Self 1

5 rows × 21 columns

In [228]:
df2['age'] = df['age']
df2['age']
Out[228]:
0      26.0
1      24.0
2      27.0
3      35.0
5      36.0
       ... 
698    27.0
699    25.0
700    34.0
702    35.0
703    26.0
Name: age, Length: 609, dtype: float64
In [229]:
df2.head()
Out[229]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score ... gender ethnicity jundice austim contry_of_res used_app_before result age_desc relation Class/ASD
0 1 1 1 1 0 0 1 1 0 0 ... 1 0 0 0 United States no no 18 and more Self 0
1 1 1 0 1 0 0 0 1 0 1 ... 0 1 0 1 Brazil no no 18 and more Self 0
2 1 1 0 1 1 0 1 1 1 1 ... 0 1 1 1 Spain no no 18 and more Parent 1
3 1 1 0 1 0 0 1 1 0 1 ... 1 0 0 1 United States no no 18 and more Self 0
5 1 1 1 1 1 0 1 1 1 1 ... 0 2 1 0 United States no no 18 and more Self 1

5 rows × 21 columns

In [230]:
## Data Visualization =>
In [231]:
# Correlation matrix : 
cor = df2.corr()
cor
Out[231]:
A1_Score A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A8_Score A9_Score A10_Score age gender ethnicity jundice austim Class/ASD
A1_Score 1.000000 0.039034 0.052606 0.129298 0.202714 0.134127 0.209795 0.135516 0.157704 0.102651 0.027540 0.087933 -0.119777 -0.008776 0.087048 0.293088
A2_Score 0.039034 1.000000 0.239698 0.185258 0.149704 0.179607 -0.003799 0.040452 0.210348 0.080901 0.009615 0.070834 -0.171501 0.147851 0.086227 0.335059
A3_Score 0.052606 0.239698 1.000000 0.404478 0.263536 0.270959 0.055974 0.001029 0.318281 0.166703 0.027012 -0.010279 -0.226991 0.051265 0.086347 0.435016
A4_Score 0.129298 0.185258 0.404478 1.000000 0.318789 0.289738 0.133785 0.008268 0.337750 0.218037 0.029078 0.046667 -0.276421 0.080995 0.196857 0.470414
A5_Score 0.202714 0.149704 0.263536 0.318789 1.000000 0.397363 0.251554 0.119779 0.393205 0.279877 -0.037472 0.050517 -0.132905 0.044450 0.088595 0.550717
A6_Score 0.134127 0.179607 0.270959 0.289738 0.397363 1.000000 0.200386 0.110421 0.481401 0.299308 0.027363 0.089600 -0.206997 0.106909 0.111966 0.606399
A7_Score 0.209795 -0.003799 0.055974 0.133785 0.251554 0.200386 1.000000 0.073314 0.187916 0.236833 -0.025192 -0.075954 -0.136536 0.008013 -0.032828 0.355280
A8_Score 0.135516 0.040452 0.001029 0.008268 0.119779 0.110421 0.073314 1.000000 0.107658 0.113028 -0.085422 -0.052451 -0.007246 -0.002781 0.034865 0.246237
A9_Score 0.157704 0.210348 0.318281 0.337750 0.393205 0.481401 0.187916 0.107658 1.000000 0.294299 0.050567 0.004407 -0.235453 0.080173 0.169547 0.641386
A10_Score 0.102651 0.080901 0.166703 0.218037 0.279877 0.299308 0.236833 0.113028 0.294299 1.000000 -0.027199 0.046025 -0.211118 0.042290 0.108180 0.377310
age 0.027540 0.009615 0.027012 0.029078 -0.037472 0.027363 -0.025192 -0.085422 0.050567 -0.027199 1.000000 0.065889 -0.102545 0.060197 0.091543 0.052350
gender 0.087933 0.070834 -0.010279 0.046667 0.050517 0.089600 -0.075954 -0.052451 0.004407 0.046025 0.065889 1.000000 -0.094235 0.023333 0.112017 0.085606
ethnicity -0.119777 -0.171501 -0.226991 -0.276421 -0.132905 -0.206997 -0.136536 -0.007246 -0.235453 -0.211118 -0.102545 -0.094235 1.000000 -0.122808 -0.178707 -0.321277
jundice -0.008776 0.147851 0.051265 0.080995 0.044450 0.106909 0.008013 -0.002781 0.080173 0.042290 0.060197 0.023333 -0.122808 1.000000 0.156428 0.128491
austim 0.087048 0.086227 0.086347 0.196857 0.088595 0.111966 -0.032828 0.034865 0.169547 0.108180 0.091543 0.112017 -0.178707 0.156428 1.000000 0.164870
Class/ASD 0.293088 0.335059 0.435016 0.470414 0.550717 0.606399 0.355280 0.246237 0.641386 0.377310 0.052350 0.085606 -0.321277 0.128491 0.164870 1.000000
In [232]:
fig, ax = plt.subplots(figsize=(10,10))         # Sample figsize in inches
dataplot = sb.heatmap(df2.corr(), cmap="YlGnBu", annot=True, linewidths=0.5, ax=ax)
In [233]:
## Choosing the most correlated featues with target feature
threshold = 0.3
a = abs(cor['Class/ASD'])
result = a[a > threshold]
result
Out[233]:
A2_Score     0.335059
A3_Score     0.435016
A4_Score     0.470414
A5_Score     0.550717
A6_Score     0.606399
A7_Score     0.355280
A9_Score     0.641386
A10_Score    0.377310
ethnicity    0.321277
Class/ASD    1.000000
Name: Class/ASD, dtype: float64
In [234]:
Columns
Out[234]:
Index(['A1_Score', 'A2_Score', 'A3_Score', 'A4_Score', 'A5_Score', 'A6_Score',
       'A7_Score', 'A8_Score', 'A9_Score', 'A10_Score', 'age', 'gender',
       'ethnicity', 'jundice', 'austim', 'contry_of_res', 'used_app_before',
       'result', 'age_desc', 'relation', 'Class/ASD'],
      dtype='object')
In [235]:
df3 = df2.drop(['A1_Score', 'A8_Score', 'A10_Score', 'age', 'gender',
       'jundice', 'austim', 'contry_of_res', 'used_app_before',
       'result', 'age_desc', 'relation'], 1)
df3.head()
Out[235]:
A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A9_Score ethnicity Class/ASD
0 1 1 1 0 0 1 0 0 0
1 1 0 1 0 0 0 0 1 0
2 1 0 1 1 0 1 1 1 1
3 1 0 1 0 0 1 0 0 0
5 1 1 1 1 0 1 1 2 1
In [236]:
# Identifying dependent(target) variable
target = df3["Class/ASD"]    # Dependent variable
target
Out[236]:
0      0
1      0
2      1
3      0
5      1
      ..
698    1
699    1
700    0
702    0
703    1
Name: Class/ASD, Length: 609, dtype: int64
In [237]:
df3
Out[237]:
A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A9_Score ethnicity Class/ASD
0 1 1 1 0 0 1 0 0 0
1 1 0 1 0 0 0 0 1 0
2 1 0 1 1 0 1 1 1 1
3 1 0 1 0 0 1 0 0 0
5 1 1 1 1 0 1 1 2 1
... ... ... ... ... ... ... ... ... ...
698 1 1 1 1 1 1 1 6 1
699 1 0 1 1 0 1 1 0 1
700 0 0 0 0 0 0 0 8 0
702 0 0 1 1 0 1 1 7 0
703 0 1 1 1 0 1 1 0 1

609 rows × 9 columns

In [238]:
# Identifying independent variables
X = df3.drop(['Class/ASD'], 1)
X
Out[238]:
A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A9_Score ethnicity
0 1 1 1 0 0 1 0 0
1 1 0 1 0 0 0 0 1
2 1 0 1 1 0 1 1 1
3 1 0 1 0 0 1 0 0
5 1 1 1 1 0 1 1 2
... ... ... ... ... ... ... ... ...
698 1 1 1 1 1 1 1 6
699 1 0 1 1 0 1 1 0
700 0 0 0 0 0 0 0 8
702 0 0 1 1 0 1 1 7
703 0 1 1 1 0 1 1 0

609 rows × 8 columns

In [239]:
#splitting the model into training and testing set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, target, test_size = 0.30, random_state = 101)
In [240]:
X_train
Out[240]:
A2_Score A3_Score A4_Score A5_Score A6_Score A7_Score A9_Score ethnicity
244 0 1 0 1 0 1 0 4
463 0 0 1 0 0 1 0 0
416 0 0 0 0 0 1 0 5
189 1 1 0 1 0 1 0 0
184 0 1 1 1 0 1 0 0
... ... ... ... ... ... ... ... ...
87 0 0 0 0 0 0 0 0
693 0 0 1 0 0 0 0 0
669 1 1 1 1 1 1 1 0
398 1 1 0 0 0 0 0 5
609 0 0 1 0 0 1 0 4

426 rows × 8 columns

In [241]:
# Training a logistics regression model
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression
from sklearn.metrics import accuracy_score

logmodel = LogisticRegression()
logmodel.fit(X_train,y_train)
predictions = logmodel.predict(X_test)
print("Accuracy = "+ str(accuracy_score(y_test,predictions)))
Accuracy = 0.9672131147540983
In [242]:
from sklearn import metrics
cm = metrics.confusion_matrix(y_test, predictions)
print(cm)
[[130   2]
 [  4  47]]
In [243]:
from sklearn.metrics import r2_score, max_error, mean_squared_error
print("R2 score = "+ str(r2_score(y_test,predictions)))
print("Max error = "+ str(max_error(y_test,predictions)))
print("mean_squared_error = "+ str(mean_squared_error(y_test,predictions)))
R2 score = 0.8368983957219251
Max error = 1
mean_squared_error = 0.03278688524590164
In [244]:
import random as random
# Including Genetic Algorithm (GA)
#defining various steps required for the genetic algorithm
def initilization_of_population(size,n_feat):
    population = []
    for i in range(size):
        chromosome = np.ones(n_feat, dtype=np.bool)
        chromosome[:int(0.3 * n_feat)] = False
        np.random.shuffle(chromosome)
        population.append(chromosome)
    return population

def fitness_score(population):
    scores = []
    for chromosome in population:
        logmodel = LogisticRegression()
        logmodel.fit(X_train.iloc[:,chromosome],y_train)
        predictions = logmodel.predict(X_test.iloc[:,chromosome])
        scores.append(accuracy_score(y_test,predictions))
    scores, population = np.array(scores), np.array(population) 
    inds = np.argsort(scores)
    return list(scores[inds][::-1]), list(population[inds,:][::-1])

def selection(pop_after_fit,n_parents):
    population_nextgen = []
    for i in range(n_parents):
        population_nextgen.append(pop_after_fit[i])
    return population_nextgen

def crossover(pop_after_sel):
    population_nextgen=pop_after_sel
    for i in range(len(pop_after_sel)):
        child=pop_after_sel[i]
        child[3:7]=pop_after_sel[(i+1)%len(pop_after_sel)][3:7]
        population_nextgen.append(child)
    return population_nextgen

def mutation(pop_after_cross,mutation_rate):
    population_nextgen = []
    for i in range(0,len(pop_after_cross)):
        chromosome = pop_after_cross[i]
        for j in range(len(chromosome)):
            if random.random() < mutation_rate:
                chromosome[j]= not chromosome[j]
        population_nextgen.append(chromosome)
    #print(population_nextgen)
    return population_nextgen

def generations(size, n_feat, n_parents, mutation_rate, n_gen, X_train, X_test, y_train, y_test):
    best_chromo= []
    best_score= []
    population_nextgen = initilization_of_population(size, n_feat)
    for i in range(n_gen):
        scores, pop_after_fit = fitness_score(population_nextgen)
        print(scores[:2])
        pop_after_sel = selection(pop_after_fit,n_parents)
        pop_after_cross = crossover(pop_after_sel)
        population_nextgen = mutation(pop_after_cross,mutation_rate)
        best_chromo.append(pop_after_fit[0])
        best_score.append(scores[0])
    return best_chromo,best_score
In [245]:
best_chromosomes, best_score = generations(size = 200, n_feat = 8, n_parents = 100, mutation_rate = 0.10,
                     n_gen = 5, X_train = X_train, X_test = X_test, y_train = y_train, y_test = y_test)
[0.9617486338797814, 0.9617486338797814]
[0.9672131147540983, 0.9672131147540983]
[0.9672131147540983, 0.9672131147540983]
[0.9672131147540983, 0.9672131147540983]
[0.9672131147540983, 0.9672131147540983]
In [246]:
# print('best_chromosomes', best_chromosomes)
print('best_score', best_score)
print(sum(best_score)/len(best_score))
best_score [0.9617486338797814, 0.9672131147540983, 0.9672131147540983, 0.9672131147540983, 0.9672131147540983]
0.966120218579235
In [247]:
# Using best chromosomes
logmodel.fit(X_train.iloc[:,best_chromosomes[-1]], y_train)
predictions_GA = logmodel.predict(X_test.iloc[:,best_chromosomes[-1]])

print("Accuracy score after genetic algorithm is = " + str(accuracy_score(y_test, predictions_GA)))
from sklearn.metrics import r2_score, max_error, mean_squared_error
print("R2 score = "+ str(r2_score(y_test,predictions_GA)))
print("Max error = "+ str(max_error(y_test,predictions_GA)))
print("mean_squared_error = "+ str(mean_squared_error(y_test,predictions_GA)))
Accuracy score after genetic algorithm is = 0.9672131147540983
R2 score = 0.8368983957219251
Max error = 1
mean_squared_error = 0.03278688524590164
In [248]:
# Decision Tree
In [249]:
from sklearn import tree
clf = tree.DecisionTreeClassifier()
clf = clf.fit(X_train, y_train)
Y_prediction = clf.predict(X_test)
tree.plot_tree(clf)
Out[249]:
[Text(146.62235915492957, 207.55636363636364, 'X[6] <= 0.5\ngini = 0.422\nsamples = 426\nvalue = [297, 129]'),
 Text(68.07992957746478, 187.7890909090909, 'X[4] <= 0.5\ngini = 0.167\nsamples = 282\nvalue = [256, 26]'),
 Text(25.345774647887325, 168.0218181818182, 'X[3] <= 0.5\ngini = 0.082\nsamples = 234\nvalue = [224, 10]'),
 Text(20.630281690140844, 148.25454545454545, 'gini = 0.0\nsamples = 156\nvalue = [156, 0]'),
 Text(30.0612676056338, 148.25454545454545, 'X[0] <= 0.5\ngini = 0.224\nsamples = 78\nvalue = [68, 10]'),
 Text(9.430985915492958, 128.48727272727274, 'X[7] <= 4.5\ngini = 0.04\nsamples = 49\nvalue = [48, 1]'),
 Text(4.715492957746479, 108.72, 'gini = 0.0\nsamples = 34\nvalue = [34, 0]'),
 Text(14.146478873239436, 108.72, 'X[5] <= 0.5\ngini = 0.124\nsamples = 15\nvalue = [14, 1]'),
 Text(9.430985915492958, 88.95272727272729, 'gini = 0.0\nsamples = 12\nvalue = [12, 0]'),
 Text(18.861971830985915, 88.95272727272729, 'X[1] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [2, 1]'),
 Text(14.146478873239436, 69.18545454545455, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(23.577464788732392, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(50.69154929577465, 128.48727272727274, 'X[2] <= 0.5\ngini = 0.428\nsamples = 29\nvalue = [20, 9]'),
 Text(33.00845070422535, 108.72, 'X[5] <= 0.5\ngini = 0.153\nsamples = 12\nvalue = [11, 1]'),
 Text(28.292957746478873, 88.95272727272729, 'gini = 0.0\nsamples = 8\nvalue = [8, 0]'),
 Text(37.72394366197183, 88.95272727272729, 'X[7] <= 3.5\ngini = 0.375\nsamples = 4\nvalue = [3, 1]'),
 Text(33.00845070422535, 69.18545454545455, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(42.43943661971831, 69.18545454545455, 'X[1] <= 0.5\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(37.72394366197183, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(47.154929577464785, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(68.37464788732395, 108.72, 'X[1] <= 0.5\ngini = 0.498\nsamples = 17\nvalue = [9, 8]'),
 Text(56.585915492957746, 88.95272727272729, 'X[7] <= 1.5\ngini = 0.278\nsamples = 6\nvalue = [5, 1]'),
 Text(51.87042253521127, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(61.30140845070422, 69.18545454545455, 'gini = 0.0\nsamples = 5\nvalue = [5, 0]'),
 Text(80.16338028169014, 88.95272727272729, 'X[5] <= 0.5\ngini = 0.463\nsamples = 11\nvalue = [4, 7]'),
 Text(70.73239436619718, 69.18545454545455, 'X[7] <= 0.5\ngini = 0.49\nsamples = 7\nvalue = [3, 4]'),
 Text(66.0169014084507, 49.418181818181836, 'gini = 0.444\nsamples = 3\nvalue = [1, 2]'),
 Text(75.44788732394366, 49.418181818181836, 'X[7] <= 1.5\ngini = 0.5\nsamples = 4\nvalue = [2, 2]'),
 Text(70.73239436619718, 29.650909090909096, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(80.16338028169014, 29.650909090909096, 'X[7] <= 3.0\ngini = 0.444\nsamples = 3\nvalue = [1, 2]'),
 Text(75.44788732394366, 9.883636363636384, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(84.87887323943661, 9.883636363636384, 'gini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(89.59436619718309, 69.18545454545455, 'X[7] <= 0.5\ngini = 0.375\nsamples = 4\nvalue = [1, 3]'),
 Text(84.87887323943661, 49.418181818181836, 'gini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(94.30985915492957, 49.418181818181836, 'gini = 0.0\nsamples = 2\nvalue = [0, 2]'),
 Text(110.81408450704225, 168.0218181818182, 'X[5] <= 0.5\ngini = 0.444\nsamples = 48\nvalue = [32, 16]'),
 Text(89.59436619718309, 148.25454545454545, 'X[2] <= 0.5\ngini = 0.198\nsamples = 27\nvalue = [24, 3]'),
 Text(84.87887323943661, 128.48727272727274, 'gini = 0.0\nsamples = 15\nvalue = [15, 0]'),
 Text(94.30985915492957, 128.48727272727274, 'X[1] <= 0.5\ngini = 0.375\nsamples = 12\nvalue = [9, 3]'),
 Text(89.59436619718309, 108.72, 'gini = 0.0\nsamples = 4\nvalue = [4, 0]'),
 Text(99.02535211267606, 108.72, 'X[3] <= 0.5\ngini = 0.469\nsamples = 8\nvalue = [5, 3]'),
 Text(94.30985915492957, 88.95272727272729, 'gini = 0.0\nsamples = 3\nvalue = [3, 0]'),
 Text(103.74084507042254, 88.95272727272729, 'X[7] <= 2.5\ngini = 0.48\nsamples = 5\nvalue = [2, 3]'),
 Text(99.02535211267606, 69.18545454545455, 'gini = 0.5\nsamples = 4\nvalue = [2, 2]'),
 Text(108.45633802816901, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(132.0338028169014, 148.25454545454545, 'X[7] <= 3.0\ngini = 0.472\nsamples = 21\nvalue = [8, 13]'),
 Text(122.60281690140845, 128.48727272727274, 'X[3] <= 0.5\ngini = 0.18\nsamples = 10\nvalue = [1, 9]'),
 Text(117.88732394366197, 108.72, 'X[7] <= 1.0\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(113.17183098591549, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(122.60281690140845, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(127.31830985915492, 108.72, 'gini = 0.0\nsamples = 8\nvalue = [0, 8]'),
 Text(141.46478873239437, 128.48727272727274, 'X[3] <= 0.5\ngini = 0.463\nsamples = 11\nvalue = [7, 4]'),
 Text(136.7492957746479, 108.72, 'gini = 0.0\nsamples = 3\nvalue = [3, 0]'),
 Text(146.18028169014084, 108.72, 'X[0] <= 0.5\ngini = 0.5\nsamples = 8\nvalue = [4, 4]'),
 Text(136.7492957746479, 88.95272727272729, 'X[2] <= 0.5\ngini = 0.375\nsamples = 4\nvalue = [3, 1]'),
 Text(132.0338028169014, 69.18545454545455, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(141.46478873239437, 69.18545454545455, 'gini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(155.6112676056338, 88.95272727272729, 'X[7] <= 6.5\ngini = 0.375\nsamples = 4\nvalue = [1, 3]'),
 Text(150.89577464788732, 69.18545454545455, 'X[1] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [1, 2]'),
 Text(146.18028169014084, 49.418181818181836, 'gini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(155.6112676056338, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(160.32676056338028, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(225.16478873239436, 187.7890909090909, 'X[3] <= 0.5\ngini = 0.407\nsamples = 144\nvalue = [41, 103]'),
 Text(183.90422535211266, 168.0218181818182, 'X[7] <= 3.5\ngini = 0.269\nsamples = 25\nvalue = [21, 4]'),
 Text(179.18873239436618, 148.25454545454545, 'X[4] <= 0.5\ngini = 0.444\nsamples = 12\nvalue = [8, 4]'),
 Text(165.04225352112675, 128.48727272727274, 'X[7] <= 2.5\ngini = 0.219\nsamples = 8\nvalue = [7, 1]'),
 Text(160.32676056338028, 108.72, 'gini = 0.0\nsamples = 6\nvalue = [6, 0]'),
 Text(169.75774647887323, 108.72, 'X[2] <= 0.5\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(165.04225352112675, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(174.4732394366197, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(193.33521126760564, 128.48727272727274, 'X[5] <= 0.5\ngini = 0.375\nsamples = 4\nvalue = [1, 3]'),
 Text(188.61971830985914, 108.72, 'X[0] <= 0.5\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(183.90422535211266, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(193.33521126760564, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(198.05070422535212, 108.72, 'gini = 0.0\nsamples = 2\nvalue = [0, 2]'),
 Text(188.61971830985914, 148.25454545454545, 'gini = 0.0\nsamples = 13\nvalue = [13, 0]'),
 Text(266.4253521126761, 168.0218181818182, 'X[4] <= 0.5\ngini = 0.28\nsamples = 119\nvalue = [20, 99]'),
 Text(233.4169014084507, 148.25454545454545, 'X[7] <= 3.5\ngini = 0.493\nsamples = 34\nvalue = [15, 19]'),
 Text(216.91267605633803, 128.48727272727274, 'X[2] <= 0.5\ngini = 0.363\nsamples = 21\nvalue = [5, 16]'),
 Text(207.48169014084507, 108.72, 'X[1] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [2, 1]'),
 Text(202.7661971830986, 88.95272727272729, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(212.19718309859155, 88.95272727272729, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(226.34366197183098, 108.72, 'X[5] <= 0.5\ngini = 0.278\nsamples = 18\nvalue = [3, 15]'),
 Text(221.6281690140845, 88.95272727272729, 'X[0] <= 0.5\ngini = 0.444\nsamples = 9\nvalue = [3, 6]'),
 Text(212.19718309859155, 69.18545454545455, 'X[1] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [2, 1]'),
 Text(207.48169014084507, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(216.91267605633803, 49.418181818181836, 'X[7] <= 0.5\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(212.19718309859155, 29.650909090909096, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(221.6281690140845, 29.650909090909096, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(231.05915492957746, 69.18545454545455, 'X[7] <= 1.5\ngini = 0.278\nsamples = 6\nvalue = [1, 5]'),
 Text(226.34366197183098, 49.418181818181836, 'gini = 0.0\nsamples = 3\nvalue = [0, 3]'),
 Text(235.77464788732394, 49.418181818181836, 'X[1] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [1, 2]'),
 Text(231.05915492957746, 29.650909090909096, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(240.4901408450704, 29.650909090909096, 'gini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(231.05915492957746, 88.95272727272729, 'gini = 0.0\nsamples = 9\nvalue = [0, 9]'),
 Text(249.92112676056337, 128.48727272727274, 'X[5] <= 0.5\ngini = 0.355\nsamples = 13\nvalue = [10, 3]'),
 Text(245.2056338028169, 108.72, 'gini = 0.0\nsamples = 6\nvalue = [6, 0]'),
 Text(254.63661971830984, 108.72, 'X[2] <= 0.5\ngini = 0.49\nsamples = 7\nvalue = [4, 3]'),
 Text(245.2056338028169, 88.95272727272729, 'X[7] <= 4.5\ngini = 0.375\nsamples = 4\nvalue = [3, 1]'),
 Text(240.4901408450704, 69.18545454545455, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(249.92112676056337, 69.18545454545455, 'X[1] <= 0.5\ngini = 0.5\nsamples = 2\nvalue = [1, 1]'),
 Text(245.2056338028169, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(254.63661971830984, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(264.0676056338028, 88.95272727272729, 'X[0] <= 0.5\ngini = 0.444\nsamples = 3\nvalue = [1, 2]'),
 Text(259.3521126760563, 69.18545454545455, 'gini = 0.0\nsamples = 2\nvalue = [0, 2]'),
 Text(268.7830985915493, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(299.4338028169014, 148.25454545454545, 'X[5] <= 0.5\ngini = 0.111\nsamples = 85\nvalue = [5, 80]'),
 Text(294.71830985915494, 128.48727272727274, 'X[1] <= 0.5\ngini = 0.302\nsamples = 27\nvalue = [5, 22]'),
 Text(278.21408450704223, 108.72, 'X[2] <= 0.5\ngini = 0.5\nsamples = 6\nvalue = [3, 3]'),
 Text(273.4985915492958, 88.95272727272729, 'gini = 0.0\nsamples = 2\nvalue = [2, 0]'),
 Text(282.92957746478874, 88.95272727272729, 'X[0] <= 0.5\ngini = 0.375\nsamples = 4\nvalue = [1, 3]'),
 Text(278.21408450704223, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(287.6450704225352, 69.18545454545455, 'gini = 0.0\nsamples = 3\nvalue = [0, 3]'),
 Text(311.2225352112676, 108.72, 'X[0] <= 0.5\ngini = 0.172\nsamples = 21\nvalue = [2, 19]'),
 Text(301.79154929577464, 88.95272727272729, 'X[7] <= 2.0\ngini = 0.32\nsamples = 5\nvalue = [1, 4]'),
 Text(297.07605633802814, 69.18545454545455, 'gini = 0.375\nsamples = 4\nvalue = [1, 3]'),
 Text(306.50704225352115, 69.18545454545455, 'gini = 0.0\nsamples = 1\nvalue = [0, 1]'),
 Text(320.65352112676055, 88.95272727272729, 'X[7] <= 1.0\ngini = 0.117\nsamples = 16\nvalue = [1, 15]'),
 Text(315.9380281690141, 69.18545454545455, 'gini = 0.0\nsamples = 11\nvalue = [0, 11]'),
 Text(325.36901408450706, 69.18545454545455, 'X[7] <= 2.5\ngini = 0.32\nsamples = 5\nvalue = [1, 4]'),
 Text(320.65352112676055, 49.418181818181836, 'gini = 0.0\nsamples = 1\nvalue = [1, 0]'),
 Text(330.0845070422535, 49.418181818181836, 'gini = 0.0\nsamples = 4\nvalue = [0, 4]'),
 Text(304.1492957746479, 128.48727272727274, 'gini = 0.0\nsamples = 58\nvalue = [0, 58]')]
In [250]:
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
confusion_matrix(y_test, Y_prediction)
print('Accuraccy using DT : ', accuracy_score(y_test, Y_prediction))
Accuraccy using DT :  0.9180327868852459
In [336]:
import random as random
# Including Genetic Algorithm (GA)
#defining various steps required for the genetic algorithm
def initilization_of_population(size,n_feat):
    population = []
    for i in range(size):
        chromosome = np.ones(n_feat, dtype=np.bool)
        chromosome[:int(0.3 * n_feat)] = False
        np.random.shuffle(chromosome)
        population.append(chromosome)
    return population

def fitness_score(population):
    scores = []
    for chromosome in population:
        clf = tree.DecisionTreeClassifier()
        clf.fit(X_train.iloc[:,chromosome],y_train)
        predictions = clf.predict(X_test.iloc[:,chromosome])
        scores.append(accuracy_score(y_test,predictions))
    scores, population = np.array(scores), np.array(population) 
    inds = np.argsort(scores)
    return list(scores[inds][::-1]), list(population[inds,:][::-1])

def selection(pop_after_fit,n_parents):
    population_nextgen = []
    for i in range(n_parents):
        population_nextgen.append(pop_after_fit[i])
    return population_nextgen

def crossover(pop_after_sel):
    population_nextgen=pop_after_sel
    for i in range(len(pop_after_sel)):
        child=pop_after_sel[i]
        child[3:7]=pop_after_sel[(i+1)%len(pop_after_sel)][3:7]
        population_nextgen.append(child)
    return population_nextgen

def mutation(pop_after_cross,mutation_rate):
    population_nextgen = []
    for i in range(0,len(pop_after_cross)):
        chromosome = pop_after_cross[i]
        for j in range(len(chromosome)):
            if random.random() < mutation_rate:
                chromosome[j]= not chromosome[j]
        population_nextgen.append(chromosome)
    #print(population_nextgen)
    return population_nextgen

def generations(size, n_feat, n_parents, mutation_rate, n_gen, X_train, X_test, y_train, y_test):
    best_chromo= []
    best_score= []
    population_nextgen = initilization_of_population(size, n_feat)
    for i in range(n_gen):
        scores, pop_after_fit = fitness_score(population_nextgen)
        print(scores[:2])
        pop_after_sel = selection(pop_after_fit,n_parents)
        pop_after_cross = crossover(pop_after_sel)
        population_nextgen = mutation(pop_after_cross,mutation_rate)
        best_chromo.append(pop_after_fit[0])
        best_score.append(scores[0])
    return best_chromo,best_score
In [337]:
best_chromosomes, best_score = generations(size = 200, n_feat = 8, n_parents = 100, mutation_rate = 0.10,
                     n_gen = 5, X_train = X_train, X_test = X_test, y_train = y_train, y_test = y_test)
[0.9508196721311475, 0.9508196721311475]
[0.9508196721311475, 0.9508196721311475]
[0.9508196721311475, 0.9508196721311475]
[0.9508196721311475, 0.9508196721311475]
[0.9508196721311475, 0.9508196721311475]
In [338]:
print(best_score)
print(sum(best_score) / len(best_score))
[0.9508196721311475, 0.9508196721311475, 0.9508196721311475, 0.9508196721311475, 0.9508196721311475]
0.9508196721311475
In [339]:
clf.fit(X_train.iloc[:,best_chromosomes[-1]], y_train)
predictions_GA = clf.predict(X_test.iloc[:,best_chromosomes[-1]])

print("Accuracy score after genetic algorithm is = " + str(accuracy_score(y_test, predictions_GA)))
from sklearn.metrics import r2_score, max_error, mean_squared_error
print("R2 score = "+ str(r2_score(y_test,predictions_GA)))
print("Max error = "+ str(max_error(y_test,predictions_GA)))
print("mean_squared_error = "+ str(mean_squared_error(y_test,predictions_GA)))
Accuracy score after genetic algorithm is = 0.9453551912568307
R2 score = 0.7281639928698752
Max error = 1
mean_squared_error = 0.0546448087431694