数据分析实战—鸢尾花数据分类

1.实战内容

(1) 加载鸢尾花数据集(iris.txt)并存到iris_df中,使用seaborn.lmplot寻找class（种类）项中的异常值，其他异常值也同时处理。

import pandas as pd
from sklearn.datasets import load_iris
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
iris=load_iris()
iris_df = pd.DataFrame(iris['data'], columns=iris['feature_names'])
iris_df['target']=iris['target']

import pandas as pd
import matplotlib.pyplot as plt
iris_df=pd.read_csv('iris.txt',sep=',')
iris_df

import seaborn as sns

import warnings
warnings.filterwarnings("ignore")

sns.lmplot(x='sepal_length',y='sepal_width',col='class',data=iris_df)
sns.lmplot(x='petal_length',y='petal_width',col='class',data=iris_df)
iris_df['class'].drop_duplicates()

#通过上面的语句，发现class中有异常值，同时发现sepal_width和sepal_length有异常值

#class应为3类，将versicolor修改为Iris-versicolor,将iris-setossa修改为Iris-setosa
iris_df.loc[iris_df['class']=='versicolor','class']='Iris-versicolor'
iris_df.loc[iris_df['class']=='Iris-setossa','class']='Iris-setosa'
sns.lmplot(x='sepal_length',y='sepal_width',col='class',data=iris_df)#重画,检验是否是3类

#通过直方图观察数据分布
iris_df.loc[iris_df['class']=='Iris-setosa','sepal_width'].hist()

# 将Iris-setosa的sepal_width小于2.5cm删除
iris_df=iris_df.loc[(iris_df['class']!='Iris-setosa')|(iris_df['sepal_width']>=2.5)]
iris_df.loc[iris_df['class']=='Iris-setosa','sepal_width'].hist()

#列出异常值
iris_df.loc[(iris_df['class']=='Iris-versicolor')&(iris_df['sepal_length']<1.0)]

# 将Iris-versicolor的sepal_length接近于0的异常值乘100，‘米’转化成‘厘米’
iris_df.loc[(iris_df['class']=='Iris-versicolor')&(iris_df['sepal_length']<1.0),'sepal_length']*= 100
iris_df.loc[iris_df['class']=='Iris-versicolor','sepal_length'].hist()

(2) 使用isnull和describe查看缺失值，并处理

# 列出缺失的样本
iris_df.isnull().sum()

iris_df.describe()

iris_df.loc[iris_df['petal_width'].isnull()]

#用该类的平均值来填补缺失值，并列出修改过样本
avg_value=iris_df.loc[iris_df['class']=='Iris-setosa','petal_width'].mean()
iris_df.loc[(iris_df['class']=='Iris-setosa')&(iris_df['petal_width'].isnull()), 'petal_width'] = avg_value
iris_df.loc[(iris_df['class']=='Iris-setosa')&(iris_df['petal_width']==avg_value)]

#检查是否还存在缺失值
iris_df.isnull().sum()

#将标签名称转化成标签(如：Iris-setosa变成0)
class_mapping={'Iris-setosa':0,'Iris-versicolor':1,'Iris-virginica':2}
iris_df['class']=iris_df['class'].map(class_mapping)
iris_df

#保存数据
iris_df.to_csv('iris-clean.csv',index=False)

(3) 导入sklearn自带的数据集load_iris,获取特征矩阵和目标数组（标签）

from sklearn.datasets import load_iris
iris=load_iris()
iris_X=iris.data
iris_Y=iris.target

(4) 使用KNeighborsClassifier()分类预测

from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split,cross_val_score
def knn_function(X,Y):
    X_train,X_test,Y_train,Y_test=train_test_split(X,Y,test_size=0.3)
    clf=KNeighborsClassifier()#建立模型
    clf.fit(X_train,Y_train)#训练模型
    predict_test=clf.predict(X_test)
    print('预测的值','\n',predict_test)
    print('真实的值','\n',Y_test)
    score=clf.score(X_test,Y_test,sample_weight=None)#计算准确率
    print('准确率','\n',score)
    return clf
knn_function(iris_X,iris_Y)

(5) 导入iris_clean.csv,获取特征矩阵和目标数组，调用函数knn_function()，保存模型

import pandas as pd
import pickle
import seaborn as sns
import matplotlib.pyplot as plt
iris= pd.read_csv('iris-clean.csv')
#获取特征矩阵和目标数组（标签）
iris_XX = iris.loc[0:,'sepal_length':'petal_width'].values
iris_YY = iris['class'].values
#调用函数
knn_model = knn_function(iris_XX,iris_YY)
# 保存模型
with open('knn_model.pkl', 'wb') as f:
     pickle.dump(knn_model, f)
# 读取保存模型
with open('knn_model.pkl', 'rb') as f:
     model = pickle.load(f)
#模型的表现与训练集的选择关系
model_accuracies = []
for repetition in range(1000):
    X_train, X_test, Y_train, Y_test = \
        train_test_split(iris_XX, iris_YY, test_size=0.3)
# 通过读取保存模型knn_model.pkl代码,建立模型model
    score = model.score(X_test, Y_test, sample_weight=None)
    model_accuracies.append(score)
sns.distplot(model_accuracies)
plt.show()

(6) 超参数与调整，以sklearn自带的鸢尾花数据为例，选择KNN模型，调整超参数K的值，用10折交叉验证判断K值为1~25时的最优值

from sklearn.model_selection import train_test_split, cross_val_score
from sklearn import datasets
from sklearn.neighbors import KNeighborsClassifier
import matplotlib.pyplot as plt

iris = datasets.load_iris()
X = iris.data
Y = iris.target
# 划分训练集和测试集，测试集占总数据的33%，随机数生成器种子为10
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33,
                                                    random_state=10)
k_range = range(1, 26)
cv_scores = []
for n in k_range:
    clf = KNeighborsClassifier(n)
    scores = cross_val_score(clf, X_train, Y_train, cv=10,scoring='accuracy') 
    cv_scores.append(scores.mean())
plt.plot(k_range, cv_scores)
plt.xlabel('K')
plt.ylabel('Accuracy')
plt.show()

#选择最优的k
best_clf = KNeighborsClassifier(n_neighbors=5)
best_clf.fit(X_train, Y_train)
print('参数',best_clf.get_params())
print('准确率',best_clf.score(X_test, Y_test))
print('预测的值',best_clf.predict(X_test))