本内容整理自coursera，欢迎交流转载。
error=num of mistakestotal number of data points accuracy=num of correcttotal number of data points
1.没有过拟合的情况 2.过拟合情况
overfitting导致large coefficient values,进而导致很大的score，这样的话使用公式 P=11+e−wTh(xi) 计算得到的概率就会特别接近于０或１，这几会导致我们对于我们的预测结果过于自信，导致即使我们已经出错的情况下我们依然坚信自己是对的！！
Total quality=ll(w)−λ||w||22
⎧⎩⎨λ=0 普通的MLE算法λ=∞ 导致w=0λ介于之间 达到平衡 由此可知，large λ 导致high bias,low variance,small λ 导致low bias,high variance. 使用梯度上升法求解 wt+1←wt+ηΔ(ll(w)−λ||w||22) 其中,Δ(ll(w)−λ||w||22)=∑Ni=1hj(xi)(1[yi=+1]−P(y=+1|xi,w))−2λw算法流程 init w=0,t=1while not converged: for j=0,1,2,…,D: wt+1j←wtj+η(∑Ni=1hj(xi)(1[yi=+1]−P(y=+1|xi,wt))−2λwtj) t=t+1
Total quality=ll(w)−λ||w||1
可以在这里下载代码和数据文件
from __future__ import division import graphlab products = graphlab.SFrame('amazon_baby_subset.gl/') # The same feature processing (same as the previous assignments) # --------------------------------------------------------------- import json with open('important_words.json', 'r') as f: # Reads the list of most frequent words important_words = json.load(f) important_words = [str(s) for s in important_words] def remove_punctuation(text): import string return text.translate(None, string.punctuation) # Remove punctuation. products['review_clean'] = products['review'].apply(remove_punctuation) # Split out the words into individual columns for word in important_words: products[word] = products['review_clean'].apply(lambda s : s.split().count(word)) train_data, validation_data = products.random_split(.8, seed=2) import numpy as np def get_numpy_data(data_sframe, features, label): data_sframe['intercept'] = 1 features = ['intercept'] + features features_sframe = data_sframe[features] feature_matrix = features_sframe.to_numpy() label_sarray = data_sframe[label] label_array = label_sarray.to_numpy() return(feature_matrix, label_array) feature_matrix_train, sentiment_train = get_numpy_data(train_data, important_words, 'sentiment') feature_matrix_valid, sentiment_valid = get_numpy_data(validation_data, important_words, 'sentiment') ''' produces probablistic estimate for P(y_i = +1 | x_i, w). estimate ranges between 0 and 1. ''' import math def predict_probability(feature_matrix, coefficients): # Take dot product of feature_matrix and coefficients ## YOUR CODE HERE socre = np.dot(feature_matrix,coefficients) # Compute P(y_i = +1 | x_i, w) using the link function ## YOUR CODE HERE predictions = 1.0/(1+math.e**(-socre)) return predictions def feature_derivative_with_L2(errors, feature, coefficient, l2_penalty, feature_is_constant): # Compute the dot product of errors and feature ## YOUR CODE HERE derivative = np.dot(errors, feature) # add L2 penalty term for any feature that isn't the intercept. if not feature_is_constant: ## YOUR CODE HERE derivative-=2*l2_penalty*coefficient return derivative def compute_log_likelihood_with_L2(feature_matrix, sentiment, coefficients, l2_penalty): indicator = (sentiment==+1) scores = np.dot(feature_matrix, coefficients) lp = np.sum((indicator-1)*scores - np.log(1. + np.exp(-scores))) - l2_penalty*np.sum(coefficients[1:]**2) return lp def logistic_regression_with_L2(feature_matrix, sentiment, initial_coefficients, step_size, l2_penalty, max_iter): coefficients = np.array(initial_coefficients) # make sure it's a numpy array for itr in xrange(max_iter): # Predict P(y_i = +1|x_i,w) using your predict_probability() function ## YOUR CODE HERE predictions = predict_probability(feature_matrix,coefficients) # Compute indicator value for (y_i = +1) indicator = (sentiment==+1) # Compute the errors as indicator - predictions errors = indicator - predictions for j in xrange(len(coefficients)): # loop over each coefficient is_intercept = (j == 0) # Recall that feature_matrix[:,j] is the feature column associated with coefficients[j]. # Compute the derivative for coefficients[j]. Save it in a variable called derivative ## YOUR CODE HERE derivative = feature_derivative_with_L2(errors,feature_matrix[:,j],coefficients[j],l2_penalty,is_intercept) # add the step size times the derivative to the current coefficient ## YOUR CODE HERE coefficients[j]+=step_size*derivative # Checking whether log likelihood is increasing if itr <= 15 or (itr <= 100 and itr % 10 == 0) or (itr <= 1000 and itr % 100 == 0) \ or (itr <= 10000 and itr % 1000 == 0) or itr % 10000 == 0: lp = compute_log_likelihood_with_L2(feature_matrix, sentiment, coefficients, l2_penalty) print 'iteration %*d: log likelihood of observed labels = %.8f' % \ (int(np.ceil(np.log10(max_iter))), itr, lp) return coefficients # run with L2 = 0 coefficients_0_penalty = logistic_regression_with_L2(feature_matrix_train, sentiment_train, initial_coefficients=np.zeros(194), step_size=5e-6, l2_penalty=0, max_iter=501)