Import Libraries¶
In [ ]:
import sqlite3
import pandas as pd
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from math import sqrt
In [ ]:
%matplotlib inline
In [3]:
from google.colab import drive
drive.mount('/content/drive')
In [ ]:
!pip install fastai==0.7.0
In [ ]:
from fastai.imports import *
from fastai.structured import *
from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor
from IPython.display import display
from sklearn import metrics
In [ ]:
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import GradientBoostingRegressor
In [ ]:
from matplotlib import rcParams
from matplotlib.cm import rainbow
import warnings
warnings.filterwarnings('ignore')
In [ ]:
Read Data from the Database into pandas¶
In [ ]:
# Create your connection.
cnx = sqlite3.connect('/content/drive/My Drive/CSV files/database.sqlite')
df = pd.read_sql_query("SELECT * FROM Player_Attributes", cnx)
In [8]:
df.head()
Out[8]:
In [9]:
print(df.shape)
df.isna().sum()
Out[9]:
In [10]:
print(df.shape)
print(df.dtypes)
df.columns
Out[10]:
Declaring the Columns we want to Use as Features¶
In [ ]:
features = [
'potential', 'crossing', 'finishing', 'heading_accuracy',
'short_passing', 'volleys', 'dribbling', 'curve', 'free_kick_accuracy',
'long_passing', 'ball_control', 'acceleration', 'sprint_speed',
'agility', 'reactions', 'balance', 'shot_power', 'jumping', 'stamina',
'strength', 'long_shots', 'aggression', 'interceptions', 'positioning',
'vision', 'penalties', 'marking', 'standing_tackle', 'sliding_tackle',
'gk_diving', 'gk_handling', 'gk_kicking', 'gk_positioning',
'gk_reflexes']
Specifying the Prediction Target
In [ ]:
target = ['overall_rating']
Cleaning the Data
In [ ]:
df = df.dropna()
In [14]:
print(df.shape)
df.head()
Out[14]:
In [ ]:
X = df[features]
In [ ]:
y = df[target]
In [17]:
print(X.shape)
print(y.shape)
In [18]:
X.head()
Out[18]:
In [ ]:
import seaborn as sns
import matplotlib.pyplot as plt
In [ ]:
plotting a heatmap to see the co-relation.¶
In [ ]:
plt.figure(1,figsize=(24,15))
sns.heatmap(X.corr(),annot=True,cmap="YlGnBu")
plt.show()
Let us look at a typical row from our features:
In [ ]:
X.iloc[2]
Out[ ]:
Displaying target values:¶
In [25]:
y.head()
Out[25]:
Split the Dataset into Training and Test Datasets¶
In [ ]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=324,shuffle=True)
In [21]:
y_test.mean()
Out[21]:
In [22]:
model1= LinearRegression(n_jobs=-1)
model1.fit(X_train, y_train)
y_pred1=model1.predict(X_test)
RMSE1 = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred1))
print(y_test.mean())
print(RMSE1)
In [24]:
len(X.columns)
Out[24]:
Performing Prediction using Decision tree Regression Model¶
In [ ]:
score=[]
for i in range(2,len(X.columns)+1):
model2= DecisionTreeRegressor(max_depth=10+i,max_features=i)
model2.fit(X_train, y_train)
y_pred2=model2.predict(X_test)
RMSE2 = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred2))
score.append(RMSE2)
In [94]:
RMSE2
Out[94]:
In [249]:
plt.figure(1,figsize=(24,8))
list1=list(range(2,len(X.columns)+1))
sns.lineplot(x=list1,y=score)
sns.scatterplot(x=list1,y=score,color='red',legend='brief')
for i in range(len(list1)):
plt.text(x =list1[i]+0.05 , y =score[i]+0.05, s =round(score[i],3), size = 10)
plt.xticks([i for i in range(2, len(X.columns) + 1)])
plt.xlabel('Max features')
plt.xlim(1,35)
plt.ylim(1.2,2.8)
plt.ylabel('Scores')
plt.title('Decision Tree Regress RMSE scores for different number of maximum features')
plt.show()
What is the mean of the expected target value in test set ?
In [ ]:
score1=[]
estimators=[1,2,4,6,8,10,12,14,18,20]
for i in estimators:
model3= RandomForestRegressor(n_jobs=-1,n_estimators=i,max_depth=5+i)
model3.fit(X_train, y_train.values.ravel())
y_pred3=model3.predict(X_test)
RMSE3 = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred3))
#print(y_test.mean())
score1.append(RMSE3)
In [ ]:
sns.set_style('whitegrid')
In [250]:
plt.figure(1,figsize=(18,8))
sns.lineplot(x=estimators,y=score1,color='yellow')
sns.scatterplot(x=estimators,y=score1,color='blue')
for i in range(len(estimators)):
plt.text(x =estimators[i]+0.05 , y =score1[i]+0.05, s =round(score1[i],3), size = 12)
plt.xticks(estimators)
plt.ylim(0.8,3.1)
#plt.xlim(1,20)
plt.xlabel('n_estimators')
plt.ylabel('Scores')
plt.title('Random Forest Tree Regress RMSE scores for different number of maximum features')
plt.show()
In [153]:
print("Computed Random Forest Scores By tuning")
score1
Out[153]:
Overall Best Random Forest Regressor Model¶
Lets do some further hyperparameter Tuning¶
- n_estimators = 20, max_depth = 25 ## let's play with max_features = [ 'auto' , 'sqrt' , 'log2' ]
- Increasing max_features generally improves the performance of the model as at each node now we have a higher number of options to be considered. However, this is not necessarily true as this decreases the diversity of individual tree which is the USP of random forest.
In [185]:
m_fea=['auto','sqrt','log2',28,34]
score_fea=[]
for i in m_fea:
model3_rf= RandomForestRegressor(n_jobs=-1,n_estimators=20,max_depth=25,max_features=i)
model3_rf.fit(X_train, y_train.values.ravel())
y_pred3_rf=model3_rf.predict(X_test)
RMSE3_rf = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred3_rf))
#print(y_test.mean())
score_fea.append(RMSE3_rf)
print('max_features :'+str(i))
print(RMSE3_rf)
In [ ]:
max_feat=pd.DataFrame({'Name':m_fea,'RMSE_Score':score_fea})
In [202]:
max_feat
Out[202]:
In [234]:
plt.figure(1,figsize=(14,8))
sns.barplot(x='Name',y='RMSE_Score',data=max_feat)
for i in range(5):
plt.text(x =i-0.1 , y = score_fea[i]+0.002, s =round(score_fea[i],3), size = 14)
plt.ylim(0.9,1.05)
plt.ylabel('RMSE Score')
plt.xlabel('max_features method')
plt.title('Comparing various max_features values on the performance of the model: less the better')
plt.show()
In [87]:
reg_rf= RandomForestRegressor(n_jobs=-1,n_estimators=1,max_depth=3,bootstrap=False)
reg_rf.fit(X_train, y_train.values.ravel())
Out[87]:
In [89]:
draw_tree(reg_rf.estimators_[0], X_train, precision=5)
In [95]:
print(RMSE1)
print(RMSE2)
print(RMSE3)
KNeighboursRegressor Not that good as shown by RMSE error¶
In [27]:
model5= KNeighborsRegressor()
model5.fit(X_train, y_train.values.ravel())
y_pred5=model5.predict(X_test)
RMSE5 = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred5))
print(y_test.mean())
print(RMSE5)
Gradient Boosting Regressor has an RMSE of 1.777¶
In [152]:
estimators_gb=[18,20,24,28,30,32,36,40,60,80,100,120]
score3=[]
for i in estimators_gb:
model6= GradientBoostingRegressor(n_estimators=i)
model6.fit(X_train, y_train.values.ravel())
y_pred6=model6.predict(X_test)
RMSE6 = sqrt(mean_squared_error(y_true = y_test, y_pred = y_pred6))
score3.append(RMSE6)
print("estimators :"+str(i))
print(RMSE6)
In [ ]:
sns.set_style('whitegrid')
In [254]:
len(score3)==len(estimators_gb)
Out[254]:
In [263]:
plt.figure(1,figsize=(18,8))
sns.lineplot(x=estimators_gb,y=score3,color='orange')
sns.scatterplot(x=estimators_gb,y=score3,color='red')
for i in range(len(estimators_gb)):
plt.text(x =estimators_gb[i]+0.05 , y =score3[i]+0.05, s =round(score3[i],3), size = 12)
#plt.xticks(estimators)
plt.ylim(1.5,3.5)
plt.xlim(1,140)
plt.xlabel('n_estimators')
plt.ylabel('Scores')
plt.title('Gradient boosting Regressor RMSE scores for different number of estimators')
Out[263]:
In [ ]:
regressor=pd.DataFrame({'Linear regression':[RMSE1],'Descicion Tree Regressor':[RMSE2],'Random Forest Regressor':[RMSE3_rf],'Gradient Boosting Regressor':[RMSE6],'K neighbours Regressor':[RMSE5]})
RMSE of different Regreesor to see has performed best¶
In [176]:
regressor
Out[176]:
#For comparision: Mean of the expected target value in test set
In [98]:
y.mean()
Out[98]:
Mean is 68.635317¶
In [ ]:
prediction=pd.DataFrame({'Test':y_test.overall_rating,'DecisionTree':y_pred2,'RandomForest':y_pred3_rf,'GradientBoosting':y_pred6})
prediction=prediction.reset_index(drop=True)
In [180]:
print(prediction.shape)
prediction.head(15)
Out[180]:
In [181]:
sns.set_style('darkgrid')
plt.figure(1,figsize=(20,8))
sns.lineplot(x=X_test.potential,y=y_test['overall_rating'],color='red',label='Test data')
sns.lineplot(x=X_test.potential,y=y_pred3,color='green',label='Random Forest Regressor')
sns.lineplot(x=X_test.potential,y=y_pred2,color='blue',label='Decision tree regresssor Predicted data')
plt.title("Plotting Prediction data vs Test data to see deviation.")
plt.legend()
plt.show()
In [ ]:
In [ ]: