In [2]:
import pandas as pd
import matplotlib.pyplot as plt
In [3]:
dataset = '../housing/housing.csv'
In [4]:
housing = pd.read_csv(dataset)
A Glance of the Dataset¶
- What you can conclude is that:
- The median income is being scaled and caped
- The housing median age and median house value are also capped, the solution:
- collect proper labels
- remove these districts from the training set
- The attributes have very different scales.
- many histograms are tail heavy
In [5]:
housing.head()
Out[5]:
In [6]:
housing.info()
In [7]:
housing.ocean_proximity.value_counts()
Out[7]:
In [8]:
housing.describe()
Out[8]:
In [9]:
housing.hist(bins=50,figsize=(20,15))
Out[9]:
Create a Test Set¶
In [10]:
import numpy as np
In [11]:
def split_train_test(data,test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size = int(len(data)*test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices],data.iloc[test_indices]
In [12]:
train_set,test_set = split_train_test(housing,0.2)
print(len(train_set),'train +',len(test_set),'test')
In [13]:
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size = 0.2,random_state=42)
To make the test set a representative of the population¶
- Random selecting test instance may introduce bias when predicting
- You can chat with experts to ensure the test set is representative of the population
- The example below use median income as an important factor which should be ensured to be represented in the test dataset.
In [14]:
housing.median_income.hist()
Out[14]:
In [15]:
housing['income_cat'] = np.ceil(housing.median_income/1.5)
housing['income_cat'].where(housing.income_cat<5,5,inplace=True)
In [16]:
housing['income_cat'].value_counts()/len(housing) #计算一个各个类别的百分比
Out[16]:
In [17]:
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1,test_size=0.2,random_state=42)
for train_index, test_index in split.split(housing,housing.income_cat):
strat_train_set = housing.loc[train_index]
strat_test_set = housing.loc[test_index]
In [18]:
strat_test_set.income_cat.value_counts()/len(strat_test_set) #可以看到，这和总体的情况非常接近
Out[18]:
In [19]:
for set in (strat_train_set,strat_test_set):
set.drop(['income_cat'],axis=1,inplace=True)
In [20]:
housing = strat_train_set.copy()
In [21]:
housing.plot(kind='scatter',x='longitude',y='latitude',alpha=0.1)
Out[21]:
In [22]:
housing.plot(kind="scatter",x='longitude',y='latitude',alpha=0.4,
s=housing['population']/100,label='population',
c='median_house_value', cmap=plt.get_cmap("jet"),colorbar=True,)
plt.legend()
Out[22]:
Looking for Correlations¶
In [23]:
corr_matrix = housing.corr()
In [24]:
corr_matrix.median_house_value
Out[24]:
In [25]:
from pandas.plotting import scatter_matrix
attributes = ['median_house_value','median_income','total_rooms','housing_median_age']
scatter_matrix(housing[attributes],figsize=(12,8))
Out[25]:
In [26]:
housing.plot(kind='scatter',x='median_income',y='median_house_value',alpha=0.1)
Out[26]:
- You can clearly see horizontal lines in the above picture
- Besides, the correlation between the income and house price is very apparent
Experimenting with Attribute Combinations¶
In [27]:
housing['rooms_per_household'] = housing.total_rooms/housing.households
housing['bedromms_per_room'] = housing.total_bedrooms/housing.total_rooms
housing['population_per_household'] = housing.population/housing.households
In [28]:
corr_matrix = housing.corr()
corr_matrix.median_house_value.sort_values(ascending=False)
Out[28]:
Prepare the data for machine learning¶
In [29]:
housing = strat_train_set.drop('median_house_value',axis=1)
housing_labels = strat_train_set.median_house_value.copy()
Data Cleaning¶
- Get rid of the corresponding districts
- Get rid of the whole attribute
- Set the values to some value(0, the mean, the median, etc,)
In [30]:
print(len(housing))
print(housing.total_bedrooms.value_counts().sum())
print('This indicates that there na values in the dataset')
#housing.dropna(subset=['total_bedrooms'])
In [31]:
print('Three ways to process missing data:')
housing.dropna(subset=['total_bedrooms'])
housing.drop('total_bedrooms',axis=1)
median = housing['total_bedrooms'].median();housing.total_bedrooms.fillna(median);
In [32]:
print('na value processor by sklearn')
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')
housing_num = housing.drop('ocean_proximity',axis=1)
In [33]:
imputer.fit(housing_num)
#下面两句等价
housing_num.median().values
imputer.statistics_
X = imputer.transform(housing_num)
housing_tr = pd.DataFrame(X,columns=housing_num.columns)
Handling text and categorical attributes¶
In [34]:
from sklearn.preprocessing import LabelEncoder
In [35]:
encoder = LabelEncoder()
housing_cat = housing.ocean_proximity
housing_cat_encoded = encoder.fit_transform(housing_cat)
housing_cat_encoded
encoder.classes_
Out[35]:
In [36]:
from sklearn.preprocessing import OneHotEncoder
encoder = OneHotEncoder()
housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1,1))
housing_cat_1hot
Out[36]:
In [37]:
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer(sparse_output=False)
housing_cat_1hot = encoder.fit_transform(housing_cat)
housing_cat_1hot
Out[37]:
Custom Transformers¶
In [38]:
from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix,population_ix,household_ix = 3,4,5,6
In [39]:
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
def __init__(self, add_bedrooms_per_room = True): # no *args or **kargs
self.add_bedrooms_per_room = add_bedrooms_per_room
def fit(self, X, y=None):
return self # nothing else to do
def transform(self, X, y=None):
rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
population_per_household = X[:, population_ix] / X[:, household_ix]
if self.add_bedrooms_per_room:
bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
return np.c_[X, rooms_per_household, population_per_household,
bedrooms_per_room]
else:
return np.c_[X, rooms_per_household, population_per_household]
In [40]:
attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
In [41]:
housing_extra_attribs = attr_adder.transform(housing.values)
Feature Scaling¶
Transformation Pipelines¶
In [42]:
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
('imputer', SimpleImputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
In [43]:
housing_num_tr = num_pipeline.fit_transform(housing_num)
In [60]:
from sklearn.base import BaseEstimator,TransformerMixin
class DataFrameSelector(BaseEstimator,TransformerMixin):
def __init__(self,attribute_names):
self.attribute_names = attribute_names
def fit(self,X,y=None):
return self
def transform(self,X,y=None):
return X[self.attribute_names].values
In [73]:
from sklearn.pipeline import FeatureUnion
num_attribs = list(housing_num)
cat_attribs = ['ocean_proximity']
num_pipeline = Pipeline(
[
('selector',DataFrameSelector(num_attribs)),
('imputer',SimpleImputer(strategy='median')),
('attribs_adder',CombinedAttributesAdder()),
('std_scaler',StandardScaler()),
]
)
cat_pipeline = Pipeline([
('selector',DataFrameSelector(cat_attribs)),
('label_binarizer',OneHotEncoder(sparse=False)),
])
full_pipeline = FeatureUnion(transformer_list=[
('num_pipeline',num_pipeline),
('cat_pipeline',cat_pipeline),
])
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared.shape
Out[73]:
In [74]:
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
('imputer', SimpleImputer(strategy="median")),
('attribs_adder', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
from sklearn.compose import ColumnTransformer
num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]
full_pipeline = ColumnTransformer([
("num", num_pipeline, num_attribs),
("cat", OneHotEncoder(), cat_attribs),
])
housing_prepared = full_pipeline.fit_transform(housing)
housing_prepared.shape
Out[74]:
Select and Train a Model¶
In [76]:
housing.head()
Out[76]:
Linear Regression¶
In [81]:
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(housing_prepared,housing_labels)
Out[81]:
In [82]:
some_data = housing.iloc[:5]
some_labels = housing_labels.iloc[:5]
some_data_prepared = full_pipeline.transform(some_data)
print("Predictions:", lin_reg.predict(some_data_prepared))
In [83]:
print(some_labels)
In [88]:
from sklearn.metrics import mean_squared_error
housing_predictions = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_labels, housing_predictions)
lin_rmse = np.sqrt(lin_mse)
lin_rmse
Out[88]:
Decision Tree Regression¶
In [95]:
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared,housing_labels)
Out[95]:
In [98]:
housing_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(housing_labels,housing_predictions)
tree_rmse = np.sqrt(tree_mse)
tree_rmse
Out[98]:
Cross Validation¶
In [104]:
from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring='neg_mean_squared_error',cv=10)
rmse_scores = np.sqrt(-scores)
print(rmse_scores,rmse_scores.mean(),rmse_scores.std())
In [108]:
lin_scores = cross_val_score(lin_reg, housing_prepared, housing_labels, scoring='neg_mean_squared_error',cv=10)
rmse_scores = np.sqrt(-lin_scores)
print(rmse_scores,rmse_scores.mean(),rmse_scores.std())
Random Forest Regressor¶
In [114]:
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared,housing_labels)
housing_predictions = forest_reg.predict(housing_prepared)
forest_mse = mean_squared_error(housing_labels,housing_predictions)
forest_rmse = np.sqrt(forest_mse)
forest_rmse
Out[114]:
In [115]:
fore_scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring='neg_mean_squared_error',cv=10)
rmse_scores = np.sqrt(-fore_scores)
print(rmse_scores,rmse_scores.mean(),rmse_scores.std())
Saving Model¶
In [116]:
from sklearn.externals import joblib
joblib.dump(my_model,'mymodel.pkl')
my_model_loaded = joblib.load('my_model.pkl')
Fine-tuning the model!¶
Grid Search¶
In [ ]:
from sklearn.model_selection import GridSearchCV
param_grid = [
# try 12 (3×4) combinations of hyperparameters
{'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},
# then try 6 (2×3) combinations with bootstrap set as False
{'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},
]
forest_reg = RandomForestRegressor(random_state=42)
# train across 5 folds, that's a total of (12+6)*5=90 rounds of training
grid_search = GridSearchCV(forest_reg, param_grid, cv=5,
scoring='neg_mean_squared_error', return_train_score=True)
grid_search.fit(housing_prepared, housing_labels)
In [ ]:
grid_search.best_params_
In [ ]:
grid_search.best_estimator_
In [ ]:
cvres = grid_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
print(np.sqrt(-mean_score), params)