Differences

This shows you the differences between two versions of the page.

--- code:model_selection [2015/10/05 13:49]
asa
+++ code:model_selection [2016/10/06 14:58] (current)
asa
@@ Line 1: / Line 1: @@
-===== model selection and cross validation in scikit-learn =====
+===== model selection in scikit-learn =====
-First let's import some modules and read in some data:
+<file python model_selection.py>
-<code python>
-In [1]: import numpy as np
+"""classifier evaluation using scikit-learn
-In [2]: from sklearn import cross_validation
+more details at:
+http://scikit-learn.org/stable/modules/cross_validation.html
+http://scikit-learn.org/stable/tutorial/statistical_inference/model_selection.html
+"""
-In [3]: from sklearn import svm
+import numpy as np
+from sklearn import cross_validation
+from sklearn import svm
+from sklearn import metrics
-In [4]: from sklearn import metrics
+# read in the heart dataset
-In [5]: data=np.genfromtxt("../data/heart_scale.data", delimiter=",")
+data=np.genfromtxt("../data/heart_scale.data", delimiter=",")
+X=data[:,1:]
+y=data[:,0]
-In [6]: X=data[:,1:]
+# first let's do regular cross-validation:
-In [7]: y=data[:,0]
+cv = cross_validation.StratifiedKFold(y, 5, shuffle=True, random_state=0)
+print (cv.test_folds)
-</code>
+classifier = svm.SVC(kernel='linear', C=1)
-The simplest form of model evaluation uses a validation/test set:
+y_predict = cross_validation.cross_val_predict(classifier, X, y, cv=cv)
+print(metrics.accuracy_score(y, y_predict))
-<code python>
-In [9]: X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.4, random_state=0)
-In [10]: classifier = svm.SVC(kernel='linear', C=1).fit(X_train, y_train)
+# grid search
-In [11]: classifier.score(X_test, y_test)
+# let's perform model selection using grid search
-Out[11]: 0.7592592592592593
+from sklearn.grid_search import GridSearchCV
+Cs = np.logspace(-2, 3, 6)
+classifier = GridSearchCV(estimator=svm.LinearSVC(), param_grid=dict(C=Cs) )
+classifier.fit(X, y)
-</code>
+# print the best accuracy, classifier and parameters:
+print (classifier.best_score_)
+print (classifier.best_estimator_)
+print (classifier.best_params_)
-Next, let'd perform cross-validation:
+# performing nested cross validation:
-<code python>
+y_predict = cross_validation.cross_val_predict(classifier, X, y, cv=cv)
+print(metrics.accuracy_score(y, y_predict))
-In [12]: scores = cross_validation.cross_val_score(classifier, X, y, cv=5, scoring='accuracy')
-In [13]:
+# if we want to do grid search over multiple parameters:
+param_grid = [
+  {'C': [1, 10, 100, 1000], 'kernel': ['linear']},
+  {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},
+ ]
+classifier = GridSearchCV(estimator=svm.SVC(), param_grid=param_grid)
-In [13]: scores = cross_validation.cross_val_score(classifier, X, y, cv=5, scoring='roc_auc')
+y_predict = cross_validation.cross_val_predict(classifier, X, y, cv=cv)
+print(metrics.accuracy_score(y, y_predict))
-In [14]: # you can also obtain the predictions by cross-validation and then compute the accuracy:
-In [15]: y_predict = cross_validation.cross_val_predict(classifier, X, y, cv=5)
-In [16]: metrics.accuracy_score(y, y_predict)
-Out[16]: 0.83703703703703702
-</code>
+</file>

CS545 fall 2016

User Tools

Site Tools

Differences

Page Tools