數據集構造:
①點狀數據
from sklearn.datasets.samples_generator import make_blobs
X, y = make_blobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60)
plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='autumn')
②圓形數據
from sklearn.datasets.samples_generator import make_circles
X,y=make_circles(100, factor=.1, noise=.1)
③正態分佈的數據
import scipy.stats as ss
norm_dist = ss.norm.rvs(size=20) #構造符合正態分佈的20個數據
ss.normaltest(norm_dist) #檢驗是否爲正態分佈
數據集劃分:
純隨機抽樣方法
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(data, test_size= , random_state= )
分層抽樣
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits= , test_size= , random_state=42)
for train_index, test_index in split.split(data, data['col']):
start_train_set = data.loc[train_index]
start_test_set = data.loc[test_index]
random函數
batch_mask = np.random.choice(train_size, batch_size)
使用np.random.choice() 可以從指定的數字中隨機選擇想要的數字。比如,np.random.choice(60000, 10)
會從0到59999之間隨機選擇10個數字。
數據集處理:
數據清理:
1 放棄相應區域 # data.dropna(subset=[“col”])
2 放棄這個屬性 # data.drop(“col”, axis=1)
3 將缺失值設置爲某個值 # data[“col”].fillna(median)
imputer 方法處理缺失值
from sklearn.processing import Imputer
imputer = Imputer(strategy="median")
imputer.fit( data['col'] ) #imputer計算了每個屬性的中位數值
X=imputer.transform( data['col'])
處理文本和分類屬性:
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()
data_encoded = encoder.fit_transform( data )
選取部分數據後,索引發生改變:
#重新定義數據索引方式
data_need=data_need.reset_index(drop=True)
流程化
from sklearn.pipeline import make_pipeline
pca =PCA(n_components=150, whiten=True, random_state=42)
svc = SVC(kernel='rbf', class_weights='balanced')
model = make_pipeline(pca, svc)
結果評估
from sklearn import metrics
#混淆矩陣
metrics.classification_report(truevalue, predicted)
#準確率:
accuracy = metrics.accuracy_score(Y_test, predict_test)
#召回率
recall = metrics.recall_score(Y_test, predict_test)
#f1指數
f1 = metrics.f1_score(Y_test, predict_test)
模型保存與加載
#模型保存
joblib.dump(lr_model, "lr.model")
#模型加載
load_lr = joblib.load("lr.model")