2.2 MiB
2.2 MiB
Машинное обучение. ВМК МГУ
In [ ]:
# !!! Данный блок будет работать только в Google-Colab !!!
! gdown 10k8Hwn9kpK9SpK4IEj4-EaWQZqgYT5-Q
! pip install -r /content/requirements_2024_25_for_colab_small.txtIn [1]:
import catboost
assert(catboost.__version__ == '1.2.7')In [3]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.simplefilter("ignore")
sns.set(style="darkgrid")
%matplotlib inlineIn [4]:
from Task import MyOneHotEncoder, SimpleCounterEncoder, FoldCountersIn [5]:
np.random.seed(1)
X = np.random.randn(100, 2)
Y = np.random.randn(100) * 1. + X[:, 0] * 3 - X[:, 1] * 0.12
Y = np.where(Y > 0, 1, 0)In [6]:
from sklearn.linear_model import LogisticRegression
clf_lbfgs = LogisticRegression(C=1, solver='lbfgs', max_iter=1, warm_start=True, fit_intercept=False)
clf_newton_cg = LogisticRegression(C=1, solver='newton-cg', max_iter=1, warm_start=True, fit_intercept=False)
coefs_lbfgs = []
coefs_nc = []
for it in range(1000):
clf_lbfgs.fit(X, Y)
coefs_lbfgs.append(clf_lbfgs.coef_[0])
clf_newton_cg.fit(X, Y)
coefs_nc.append(clf_newton_cg.coef_[0])
coefs_lbfgs = np.array(coefs_lbfgs)
coefs_nc = np.array(coefs_nc)In [7]:
x1 = np.linspace(0.5, 3.5, 1000)
x2 = np.linspace(-0.5, -0.1, 1000)
X1, X2 = np.meshgrid(x1, x2)
def loss(X1, X2, X, Y):
def sigmoid(x):
return 1 / (1 + np.exp(-x))
p = sigmoid(X1[None] * X[:, 0, None, None] + X2[None] * X[:, 1, None, None]) # 100, 200, 200
return -np.sum(Y[:, None, None] * np.log(p) + (1 - Y[:, None, None]) * np.log(1 - p), axis=0) + 0.5 * (X1**2 + X2**2)
Z = loss(X1, X2, X, Y)In [8]:
sns.set(style="darkgrid")
plt.figure(figsize=(20, 10))
plt.contour(X1, X2, Z, levels=100)
plt.plot(coefs_lbfgs[:, 0], coefs_lbfgs[:, 1], color="green", linewidth=4, label="LBFGS")
plt.scatter(coefs_lbfgs[:, 0], coefs_lbfgs[:, 1], color="green", s=60)
plt.plot(coefs_nc[:, 0], coefs_nc[:, 1], color="red", linewidth=4, label="Newton-cg")
plt.scatter(coefs_nc[:, 0], coefs_nc[:, 1], color="red", s=60)
plt.legend(fontsize=25)
plt.xlabel("x1", size=15)
plt.ylabel("x2", size=15)
plt.title("Сходимость методов к оптимальному значению", size=25)
plt.show()
In [9]:
np.random.seed(1)
X = np.hstack((np.random.randn(100, 1), np.random.uniform(7, 12, (100, 1))))
Y = np.random.randn(100) * 1. + X[:, 0] * 3 - X[:, 1] * 0.12
Y = np.where(Y > 0, 1, 0)In [10]:
clf_lbfgs = LogisticRegression(C=1, solver='lbfgs', max_iter=1, warm_start=True, fit_intercept=False)
clf_newton_cg = LogisticRegression(C=1, solver='newton-cg', max_iter=1, warm_start=True, fit_intercept=False)
coefs_lbfgs = []
coefs_nc = []
for it in range(1000):
clf_lbfgs.fit(X, Y)
coefs_lbfgs.append(clf_lbfgs.coef_[0])
clf_newton_cg.fit(X, Y)
coefs_nc.append(clf_newton_cg.coef_[0])
coefs_lbfgs = np.array(coefs_lbfgs)
coefs_nc = np.array(coefs_nc)
x1 = np.linspace(0, 3.5, 1000)
x2 = np.linspace(0.01, -0.1, 1000)
X1, X2 = np.meshgrid(x1, x2)
Z = loss(X1, X2, X, Y)In [11]:
sns.set(style="darkgrid")
plt.figure(figsize=(20, 10))
plt.contour(X1, X2, Z, levels=100)
plt.plot(coefs_lbfgs[:, 0], coefs_lbfgs[:, 1], color="green", linewidth=4, label="LBFGS")
plt.scatter(coefs_lbfgs[:, 0], coefs_lbfgs[:, 1], color="green", s=60)
plt.plot(coefs_nc[:, 0], coefs_nc[:, 1], color="red", linewidth=4, label="Newton-cg")
plt.scatter(coefs_nc[:, 0], coefs_nc[:, 1], color="red", s=60)
plt.legend(fontsize=25)
plt.xlabel("x1", size=15)
plt.ylabel("x2", size=15)
plt.title("Сходимость методов к оптимальному значению", size=25)
plt.show()
In [12]:
X1 = []
X2 = []
for i in range(100):
np.random.seed(1)
x1 = np.random.uniform(0, 5)
x2 = np.random.uniform(0, 5)
X1.append(x1)
X2.append(x2)
X1 = np.array(X1)
X2 = np.array(X2)
X = np.hstack((X1[:, None], X2[:, None]))
Y = []
for i in range(100):
p = 1 / (1 + np.exp(-(X1[i] + X2[i] - 5)))
y = np.random.choice([0, 1], p = [1 - p, p])
Y.append(y)
Y = np.array(Y)In [13]:
!gdown 1AgUMxgMK-eRjzthevCk9g-J_s2vpBFpeDownloading... From: https://drive.google.com/uc?id=1AgUMxgMK-eRjzthevCk9g-J_s2vpBFpe To: C:\Users\mozhu\PycharmProjects\ML_2024\Task6\Research\Notebook\weatherAUS.csv 0%| | 0.00/14.1M [00:00<?, ?B/s] 4%|3 | 524k/14.1M [00:00<00:08, 1.57MB/s] 11%|#1 | 1.57M/14.1M [00:00<00:03, 3.86MB/s] 19%|#8 | 2.62M/14.1M [00:00<00:02, 4.49MB/s] 30%|##9 | 4.19M/14.1M [00:00<00:01, 6.27MB/s] 45%|####4 | 6.29M/14.1M [00:00<00:00, 8.87MB/s] 56%|#####5 | 7.86M/14.1M [00:01<00:00, 9.79MB/s] 67%|######6 | 9.44M/14.1M [00:01<00:00, 10.1MB/s] 82%|########1 | 11.5M/14.1M [00:01<00:00, 10.7MB/s] 93%|#########2| 13.1M/14.1M [00:01<00:00, 10.9MB/s] 100%|##########| 14.1M/14.1M [00:01<00:00, 8.83MB/s]
In [15]:
df = pd.read_csv("weatherAUS.csv")
df.head(5)Out [15]:
| Date | Location | MinTemp | MaxTemp | Rainfall | Evaporation | Sunshine | WindGustDir | WindGustSpeed | WindDir9am | ... | Humidity9am | Humidity3pm | Pressure9am | Pressure3pm | Cloud9am | Cloud3pm | Temp9am | Temp3pm | RainToday | RainTomorrow | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 2008-12-01 | Albury | 13.4 | 22.9 | 0.6 | NaN | NaN | W | 44.0 | W | ... | 71.0 | 22.0 | 1007.7 | 1007.1 | 8.0 | NaN | 16.9 | 21.8 | No | No |
| 1 | 2008-12-02 | Albury | 7.4 | 25.1 | 0.0 | NaN | NaN | WNW | 44.0 | NNW | ... | 44.0 | 25.0 | 1010.6 | 1007.8 | NaN | NaN | 17.2 | 24.3 | No | No |
| 2 | 2008-12-03 | Albury | 12.9 | 25.7 | 0.0 | NaN | NaN | WSW | 46.0 | W | ... | 38.0 | 30.0 | 1007.6 | 1008.7 | NaN | 2.0 | 21.0 | 23.2 | No | No |
| 3 | 2008-12-04 | Albury | 9.2 | 28.0 | 0.0 | NaN | NaN | NE | 24.0 | SE | ... | 45.0 | 16.0 | 1017.6 | 1012.8 | NaN | NaN | 18.1 | 26.5 | No | No |
| 4 | 2008-12-05 | Albury | 17.5 | 32.3 | 1.0 | NaN | NaN | W | 41.0 | ENE | ... | 82.0 | 33.0 | 1010.8 | 1006.0 | 7.0 | 8.0 | 17.8 | 29.7 | No | No |
5 rows × 23 columns
In [16]:
df.shapeOut [16]:
(145460, 23)
In [17]:
df.columnsOut [17]:
Index(['Date', 'Location', 'MinTemp', 'MaxTemp', 'Rainfall', 'Evaporation',
'Sunshine', 'WindGustDir', 'WindGustSpeed', 'WindDir9am', 'WindDir3pm',
'WindSpeed9am', 'WindSpeed3pm', 'Humidity9am', 'Humidity3pm',
'Pressure9am', 'Pressure3pm', 'Cloud9am', 'Cloud3pm', 'Temp9am',
'Temp3pm', 'RainToday', 'RainTomorrow'],
dtype='object')In [18]:
df['RainTomorrow'].unique()Out [18]:
array(['No', 'Yes', nan], dtype=object)
In [19]:
df = df[df['RainTomorrow'] == df['RainTomorrow']]
df['RainTomorrow'].unique()Out [19]:
array(['No', 'Yes'], dtype=object)
In [20]:
df['RainTomorrow'] = df['RainTomorrow'].map({'Yes': 1., 'No': 0.})
df['RainToday'] = df['RainToday'].map({'Yes': 1., 'No': 0.})In [21]:
print(df.shape)(142193, 23)
In [22]:
df.info()<class 'pandas.core.frame.DataFrame'> Index: 142193 entries, 0 to 145458 Data columns (total 23 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 Date 142193 non-null object 1 Location 142193 non-null object 2 MinTemp 141556 non-null float64 3 MaxTemp 141871 non-null float64 4 Rainfall 140787 non-null float64 5 Evaporation 81350 non-null float64 6 Sunshine 74377 non-null float64 7 WindGustDir 132863 non-null object 8 WindGustSpeed 132923 non-null float64 9 WindDir9am 132180 non-null object 10 WindDir3pm 138415 non-null object 11 WindSpeed9am 140845 non-null float64 12 WindSpeed3pm 139563 non-null float64 13 Humidity9am 140419 non-null float64 14 Humidity3pm 138583 non-null float64 15 Pressure9am 128179 non-null float64 16 Pressure3pm 128212 non-null float64 17 Cloud9am 88536 non-null float64 18 Cloud3pm 85099 non-null float64 19 Temp9am 141289 non-null float64 20 Temp3pm 139467 non-null float64 21 RainToday 140787 non-null float64 22 RainTomorrow 142193 non-null float64 dtypes: float64(18), object(5) memory usage: 26.0+ MB
In [23]:
from sklearn.model_selection import train_test_splitIn [24]:
y = df.RainTomorrow
X = df.drop(columns=["RainTomorrow"])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=2024)In [25]:
X_train.shapeOut [25]:
(99535, 22)
In [26]:
numeric_data = X_train.select_dtypes([np.number])
numeric_data_median = numeric_data.median()
numeric_features = numeric_data.columns
X_train = X_train.fillna(numeric_data_median)
X_test = X_test.fillna(numeric_data_median)In [27]:
len(numeric_features)Out [27]:
17
In [28]:
correlations = X_train[numeric_features].corrwith(y_train).sort_values(ascending=False)
plot = sns.barplot(y=correlations.index, x=correlations)
plot.set_title("Корреляции между вещественными признаками и целевой переменной", size=15)
plot.figure.set_size_inches(17, 10)
In [43]:
from sklearn.metrics import log_loss, roc_auc_scoreIn [44]:
model = LogisticRegression(solver='lbfgs', max_iter=1000)
model.fit(X_train[numeric_features], y_train)Out [44]:
LogisticRegression(max_iter=1000)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
LogisticRegression(max_iter=1000)
In [45]:
y_pred = model.predict_proba(X_test[numeric_features])[:, 1]
y_train_pred = model.predict_proba(X_train[numeric_features])[:, 1]
print("Test logloss = %.4f" % log_loss(y_test, y_pred))
print("Train logloss = %.4f" % log_loss(y_train, y_train_pred))
print("Test roc auc score = %.4f" % roc_auc_score(y_test, y_pred))
print("Train roc auc score = %.4f" % roc_auc_score(y_train, y_train_pred))Test logloss = 0.3635 Train logloss = 0.3695 Test roc auc score = 0.8603 Train roc auc score = 0.8564
In [46]:
model.n_iter_Out [46]:
array([1000])
In [39]:
plt.figure(figsize=(7, 7))
sorted_weights = sorted(zip(model.coef_[0], numeric_features), reverse=True)
weights = [x[0] for x in sorted_weights]
features = [x[1] for x in sorted_weights]
_ = sns.barplot(y=features, x=weights).set_title("Гистограмма весов", size=15)
In [40]:
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train[numeric_features])
X_test_scaled = scaler.transform(X_test[numeric_features])In [41]:
model = LogisticRegression(solver='lbfgs', max_iter=1000)
model.fit(X_train_scaled, y_train)Out [41]:
LogisticRegression(max_iter=1000)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
LogisticRegression(max_iter=1000)
In [42]:
model.n_iter_Out [42]:
array([27])
In [47]:
y_pred = model.predict_proba(X_test_scaled)[:, 1]
y_train_pred = model.predict_proba(X_train_scaled)[:, 1]
print("Test logloss = %.4f" % log_loss(y_test, y_pred))
print("Train logloss = %.4f" % log_loss(y_train, y_train_pred))
print("Test roc auc score = %.4f" % roc_auc_score(y_test, y_pred))
print("Train roc auc score = %.4f" % roc_auc_score(y_train, y_train_pred))Test logloss = 0.6354 Train logloss = 0.6358 Test roc auc score = 0.8050 Train roc auc score = 0.8030
In [48]:
#ВНИМАНИЕ!!! Эта ячейка может считаться несколько минут, не пугайтесь)
from sklearn.model_selection import GridSearchCV
C = np.logspace(-2, 5, 40)
searcher = GridSearchCV(LogisticRegression(solver='lbfgs'), [{"C": C}], scoring="roc_auc", cv=10)
searcher.fit(X_train_scaled, y_train)
best_C = searcher.best_params_["C"]
print("Best C = %.4f" % best_C)
_ = plt.plot(C, searcher.cv_results_["mean_test_score"])
plt.xscale("log")
plt.xlabel("C")
plt.ylabel("CV score")
plt.show()Best C = 11.2534
In [49]:
from sklearn.pipeline import Pipeline
simple_pipeline = Pipeline([
('scaling', StandardScaler()),
('classification', LogisticRegression(solver='lbfgs', C=best_C))
])
model = simple_pipeline.fit(X_train[numeric_features], y_train)
y_pred = model.predict_proba(X_test[numeric_features])[:, 1]
print("Test logloss = %.4f" % log_loss(y_test, y_pred))
print("Test roc auc score = %.4f" % roc_auc_score(y_test, y_pred))Test logloss = 0.3589 Test roc auc score = 0.8652
In [52]:
model1 = LogisticRegression(solver='lbfgs', max_iter=1000)
model1.fit(X_train_scaled, y_train)
y_pred = model1.predict_proba(X_test_scaled)[:, 1]
y_train_pred = model1.predict_proba(X_train_scaled)[:, 1]
print("Logloss = %.6f" % log_loss(y_test, y_pred))
print("Roc auc score = %.6f" % roc_auc_score(y_test, y_pred))
print("Значение C без GridSearch:", model1.C)
simple_pipeline = Pipeline([
('scaling', StandardScaler()),
('classification', LogisticRegression(solver='lbfgs', C=best_C))
])
print("________________________________"*3)
model_2 = simple_pipeline.fit(X_train[numeric_features], y_train)
y_pred = model_2.predict_proba(X_test[numeric_features])[:, 1]
print("Logloss = %.6f" % log_loss(y_test, y_pred))
print("Roc auc score = %.6f" % roc_auc_score(y_test, y_pred))
print("Значение C с GridSearch:", model_2.named_steps['classification'].C)Logloss = 0.358869
Roc auc score = 0.865176
Значение C без GridSearch: 1.0
________________________________________________________________________________________________
Logloss = 0.358868
Roc auc score = 0.865177
Значение C с GridSearch: 11.253355826007645
In [53]:
categorical = list(X_train.drop(columns=["Date"]).dtypes[X_train.dtypes == "object"].index)
X_train[categorical] = X_train[categorical].fillna("NotGiven")
X_test[categorical] = X_test[categorical].fillna("NotGiven")In [54]:
from sklearn.compose import ColumnTransformer
column_transformer = ColumnTransformer([
('ohe', MyOneHotEncoder(), categorical),
('scaling', StandardScaler(), numeric_features)
])
pipeline = Pipeline(steps=[
('ohe', column_transformer),
('classification', LogisticRegression(solver='lbfgs', max_iter=200))
])
model = pipeline.fit(X_train.drop(columns=["Date"]), y_train)
y_pred = model.predict_proba(X_test.drop(columns=["Date"]))[:, 1]
print("Test logloss = %.4f" % log_loss(y_test, y_pred))
print("Test roc auc score = %.4f" % roc_auc_score(y_test, y_pred))Test logloss = 0.3498 Test roc auc score = 0.8721
Warning:
Output truncated. This notebook contains too many cells to display efficiently.