Регуляризованная логистическая регрессия в Python (Andrew ng Course)

Question

Я начинаю путешествие по ML, и у меня возникают проблемы с этим упражнением по кодированию вот мой код

import numpy as np
import pandas as pd
import scipy.optimize as op

# Read the data and give it labels
data = pd.read_csv('ex2data2.txt', header=None, name['Test1', 'Test2', 'Accepted'])

# Separate the features to make it fit into the mapFeature function
X1 = data['Test1'].values.T
X2 = data['Test2'].values.T

# This function makes more features (degree)
def mapFeature(x1, x2):
    degree = 6
    out = np.ones((x1.shape[0], sum(range(degree + 2))))
    curr_column = 1
    for i in range(1, degree + 1):
        for j in range(i+1):
            out[:,curr_column] = np.power(x1, i-j) * np.power(x2, j)
            curr_column += 1
    return out


# Separate the data into training and target, also initialize theta
X = mapFeature(X1, X2)
y = np.matrix(data['Accepted'].values).T
m, n = X.shape
cols = X.shape[1]
theta = np.matrix(np.zeros(cols))

#Initialize the learningRate(sigma)
learningRate = 1


# Define the Sigmoid Function (Output between 0 and 1)
def sigmoid(z):
    return 1 / (1 + np.exp(-z))


def cost(theta, X, y, learningRate):
    # This is require to make the optimize function work
    theta = theta.reshape(-1, 1)
    error = sigmoid(X @ theta)
    first = np.multiply(-y, np.log(error))
    second = np.multiply(1 - y, np.log(1 - error))
    j = np.sum((first - second)) / m + (learningRate * np.sum(np.power(theta, 2)) / 2 * m)
    return j


# Define the gradient of the cost function
def gradient(theta, X, y, learningRate):
    # This is require to make the optimize function work
    theta = theta.reshape(-1, 1)
    error = sigmoid(X @ theta)
    grad =  (X.T @ (error - y)) / m + ((learningRate * theta) / m)
    grad_no = (X.T @ (error - y)) / m
    grad[0] = grad_no[0]
    return grad


Result = op.minimize(fun=cost, x0=theta, args=(X, y, learningRate), method='TNC', jac=gradient)
opt_theta = np.matrix(Result.x)


def predict(theta, X):
    sigValue = sigmoid(X @ theta.T)
    p = sigValue >= 0.5
    return p

p = predict(opt_theta, X)
print('Train Accuracy: {:f}'.format(np.mean(p == y) * 100))

Итак, когда learningRate = 1, точность должна быть около 83,05%, но я получаю 80.5%, а когда learningRate = 0, точность должна быть 91.52%, но я получаю 87.28%

Итак, вопрос Что я делаю не так? Почему моя точность ниже проблемного ответа по умолчанию?

Надеюсь, кто-то может направить меня в правильном направлении. Спасибо!

П.Д .: Вот набор данных, может быть, он может помочь

https://raw.githubusercontent.com/TheGirlWhiteWithBandages/Machine-Learning-Algorithms/master/Logistic%20Regression/ex2data2.txt

Answer 1

Эй, ребята, я нашел способ сделать это еще лучше! Вот код

import numpy as np
import pandas as pd
import scipy.optimize as op
from sklearn.preprocessing import PolynomialFeatures

# Read the data and give it labels
data = pd.read_csv('ex2data2.txt', header=None, names=['Test1', 'Test2', 'Accepted'])
# Separate the data into training and target
X = (data.iloc[:, 0:2]).values
y = (data.iloc[:, 2:3]).values
# Modify the features to a certain degree (Polynomial)
poly = PolynomialFeatures(6)
m = y.size
XX = poly.fit_transform(data.iloc[:, 0:2].values)
# Initialize Theta
theta = np.zeros(XX.shape[1])


# Define the Sigmoid Function (Output between 0 and 1)
def sigmoid(z):
    return(1 / (1 + np.exp(-z)))


# Define the Regularized cost function
def costFunctionReg(theta, reg, *args):
    # This is require to make the optimize function work
    h = sigmoid(XX @ theta)
    first = np.log(h).T @ - y
    second = np.log(1 - h).T @ (1 - y)
    J = (1 / m) * (first - second) + (reg / (2 * m)) * np.sum(np.square(theta[1:]))
    return J


# Define the Regularized gradient function
def gradientReg(theta, reg, *args):
    theta = theta.reshape(-1, 1)
    h = sigmoid(XX @ theta)
    grad = (1 / m) * (XX.T @ (h - y)) + (reg / m) * np.r_[[[0]], theta[1:]]
    return grad.flatten()


# Define the predict Function
def predict(theta, X):
    sigValue = sigmoid(X @ theta.T)
    p = sigValue >= 0.5
    return p


# A loop to test between different values for sigma (reg parameter)
for i, Sigma in enumerate([0, 1, 100]):
    # Optimize costFunctionReg 
    res2 = op.minimize(costFunctionReg, theta, args=(Sigma, XX, y), method=None, jac=gradientReg)

    # Get the accuracy of the model
    accuracy = 100 * sum(predict(res2.x, XX) == y.ravel()) / y.size

    # Get the Error between different weights
    error1 = costFunctionReg(res2.x, Sigma, XX, y)

    # print the accuracy and error
    print('Train accuracy {}% with Lambda = {}'.format(np.round(accuracy, decimals=4), Sigma))
    print(error1)

Спасибо за вашу помощь!

Answer 2

попробуйте это:

# import library
import pandas as pd
import numpy as np
dataset = pd.read_csv('ex2data2.csv',names = ['Test #1','Test #2','Accepted'])


# splitting to x and y variables for features and target variable
x = dataset.iloc[:,:-1].values
y = dataset.iloc[:,-1].values
print('x[0] ={}, y[0] ={}'.format(x[0],y[0]))
m, n = x.shape
print('#{} Number of training samples, #{} features per sample'.format(m,n))


# import library FeatureMapping
from sklearn.preprocessing import PolynomialFeatures

# We also add one column of ones to interpret theta 0 (x with power of 0 = 1) by 
include_bias as True
pf = PolynomialFeatures(degree = 6, include_bias = True)
x_poly = pf.fit_transform(x)
pd.DataFrame(x_poly).head(5)


m,n = x_poly.shape

# define theta as zero
theta = np.zeros(n)

# define hyperparameter λ
lambda_ = 1

# reshape (-1,1) because we just have one feature in y column
y = y.reshape(-1,1)


def sigmoid(z):
    return 1/(1+np.exp(-z))
def lr_hypothesis(x,theta):
    return np.dot(x,theta)

def compute_cost(theta,x,y,lambda_):
    theta = theta.reshape(n,1)
    infunc1 = -y*(np.log(sigmoid(lr_hypothesis(x,theta)))) - ((1-y)*(np.log(1 - sigmoid(lr_hypothesis(x,theta)))))
    infunc2 = (lambda_*np.sum(theta[1:]**2))/(2*m)
    j = np.sum(infunc1)/m+ infunc2
    return j

# gradient[0] correspond to gradient for theta(0)
# gradient[1:] correspond to gradient for theta(j) j>0
def compute_gradient(theta,x,y,lambda_):
    gradient = np.zeros(n).reshape(n,)
    theta = theta.reshape(n,1)
    infunc1 = sigmoid(lr_hypothesis(x,theta))-y
    gradient_in = np.dot(x.transpose(),infunc1)/m
    gradient[0] = gradient_in[0,0] # theta(0)
    gradient[1:] = gradient_in[1:,0]+(lambda_*theta[1:,]/m).reshape(n-1,) # theta(j) ; j>0
    gradient = gradient.flatten()
    return gradient

Теперь вы можете проверить свою стоимость и градиент без оптимизации. Код ниже будет оптимизировать модель:

# hyperparameters
m,n = x_poly.shape

# define theta as zero
theta = np.zeros(n)

# define hyperparameter λ
lambda_array = [0, 1, 10, 100]


import scipy.optimize as opt

for i in range(0,len(lambda_array)):
    # Train
    print('======================================== Iteration {} ===================================='.format(i))
    optimized = opt.minimize(fun = compute_cost, x0 = theta, args = (x_poly, y,lambda_array[i]),
                         method = 'TNC', jac = compute_gradient)
    new_theta = optimized.x

    # Prediction
    y_pred_train = predictor(x_poly,new_theta)
    cm_train = confusion_matrix(y,y_pred_train)
    t_train,f_train,acc_train = acc(cm_train)
    print('With lambda = {}, {} correct, {} wrong ==========> accuracy = {}%'
      .format(lambda_array[i],t_train,f_train,acc_train*100))

Теперь вы должны увидеть вывод:

 === Iteration 0 === With lambda = 0, 104 correct, 14 wrong ==========> accuracy = 88.13559322033898%
 === Iteration 1 === With lambda = 1, 98 correct, 20 wrong ==========> accuracy = 83.05084745762711%
 === Iteration 2 === With lambda = 10, 88 correct, 30 wrong ==========> accuracy = 74.57627118644068%
 === Iteration 3 === With lambda = 100, 72 correct, 46 wrong ==========> accuracy = 61.016949152542374%