civisanalytics / civisml-extensions Goto Github PK

for train, test in cv.split(X, y):
    for name, est in self.estimator_list[:-1]:
        # adapted from sklearn.model_selection._fit_and_predict
        # Adjust length of sample weights
        fit_params_est_adjusted = dict([
            (k, _index_param_value(X, v, train))
            for k, v in fit_params_ests[name].items()])

        # Fit estimator on training set and score out-of-sample
        _jobs.append(delayed(_fit_predict)(
            clone(est),
            X[train],
            y[train],
            X[test],
            **fit_params_est_adjusted))

# Extract the results from joblib
Xmeta, ymeta = None, None
for train, test in cv.split(X, y):
    ybase = np.empty((y[test].shape[0], 0))
    for name, est in self.estimator_list[:-1]:
        # Build design matrix out of out-of-sample predictions
        ybase = np.hstack((ybase, _out.pop(0)))

    # Append the test outputs to what will eventually be the features
    # for the meta-estimator.
    if Xmeta is not None:
        ymeta = np.concatenate((ymeta, y[test]))
        Xmeta = np.vstack((Xmeta, ybase))
    else:
        Xmeta = ybase
        ymeta = y[test]

~/.virtualenvs/dsmodels/lib/python3.7/site-packages/sklearn/metrics/ranking.py in roc_auc_score(y_true, y_score, average, sample_weight, max_fpr)
    354     return _average_binary_score(
    355         _binary_roc_auc_score, y_true, y_score, average,
--> 356         sample_weight=sample_weight)
    357 
    358 

~/.virtualenvs/dsmodels/lib/python3.7/site-packages/sklearn/metrics/base.py in _average_binary_score(binary_metric, y_true, y_score, average, sample_weight)
     72     y_type = type_of_target(y_true)
     73     if y_type not in ("binary", "multilabel-indicator"):
---> 74         raise ValueError("{0} format is not supported".format(y_type))
     75 
     76     if y_type == "binary":

ValueError: multiclass format is not supported

from civismlext.stacking import StackedRegressor
from civismlext.nonnegative import NonNegativeLinearRegression

def create_model():
    model = Sequential()
    model.add(Dense(150, activation='softmax', kernel_initializer='VarianceScaling', input_dim=456, name='HL1'))
    model.add(Dropout(0.25, name="Dropout1"))
    model.add(Dense(150, kernel_initializer='VarianceScaling', activation='softmax', name='HL2'))
    model.add(Dropout(0.25, name="Dropout2"))
    model.add(Dense(1, name='Output_Layer'))
    model.compile(optimizer='adam', loss='mae', metrics=['mae', 'mean_squared_error'])
    return model

mlp_model = KerasRegressor(build_fn=create_model, epochs=50, batch_size=75, validation_split=0.2, verbose=True)

super_learner = StackedRegressor([
    ('pipe_mlp', mlp_model),
    ('rf', rf),
    ('xgb', gb),
    ('meta', NonNegativeLinearRegression())
], cv=5, n_jobs=2, verbose=5)

MaybeEncodingError                        Traceback (most recent call last)
<ipython-input-7-1d4b04377633> in <module>()
      1 # fitting the model
----> 2 super_learner.fit(X_train[:50], y_train[:50])

~/anaconda3/lib/python3.6/site-packages/civismlext/stacking.py in fit(self, X, y, **fit_params)
    163         self.meta_estimator.fit(Xmeta, ymeta, **meta_params)
    164         # Now fit base estimators again, this time on full training set
--> 165         self._base_est_fit(X, y, **fit_params)
    166 
    167         return self

~/anaconda3/lib/python3.6/site-packages/civismlext/stacking.py in _base_est_fit(self, X, y, **fit_params)
    220             n_jobs=self.n_jobs,
    221             verbose=self.verbose,
--> 222             pre_dispatch=self.pre_dispatch)(_jobs)
    223 
    224         for name, _ in self.estimator_list[:-1]:

~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in __call__(self, iterable)
    787                 # consumption.
    788                 self._iterating = False
--> 789             self.retrieve()
    790             # Make sure that we get a last message telling us we are done
    791             elapsed_time = time.time() - self._start_time

~/anaconda3/lib/python3.6/site-packages/sklearn/externals/joblib/parallel.py in retrieve(self)
    697             try:
    698                 if getattr(self._backend, 'supports_timeout', False):
--> 699                     self._output.extend(job.get(timeout=self.timeout))
    700                 else:
    701                     self._output.extend(job.get())

~/anaconda3/lib/python3.6/multiprocessing/pool.py in get(self, timeout)
    642             return self._value
    643         else:
--> 644             raise self._value
    645 
    646     def _set(self, i, obj):

MaybeEncodingError: Error sending result: '[<keras.callbacks.History object at 0x7f93fe43c7b8>]'. Reason: 'TypeError("can't pickle _thread.lock objects",)'

import numpy as np

import xgboost as xgb

from scipy.stats import randint as sp_randint
from scipy.stats import uniform as sp_float


# XGBoost with Hyperband Hyperparameter Optimization
clf = xgb.XGBRegressor()

clf.set_params(**{"n_jobs": 4})

# Hyperparameter search boundaries
param_grid = {
              # Parameters for Tree Booster
              'eta': sp_float(0, 1),
              'gamma': sp_randint(0, 100),
              'max_depth': sp_randint(1, 3),
              'learning_rate': sp_float(.001, .005),
              'n_estimators': sp_randint(5000, 40000),

              'min_child_weight': sp_randint(0, 50),
              'max_delta_step': sp_randint(0, np.log(upper_limit)),
              'subsample': sp_float(0, 1),

              # Family of parameters for subsampling of columns
              'colsample_bytree': sp_float(0.2, 1),
              'colsample_bylevel': sp_float(0.2, 1),
              'colsample_bynode': sp_float(0.2, 1),

              # Regularization Params
              'lambda': sp_randint(1, 10),
              'alpha': sp_randint(0, 100),
              }

from civismlext.hyperband import HyperbandSearchCV
    tuned_model = HyperbandSearchCV(regressor,
                                    param_distributions=param_grid,
                                    cost_parameter_max={'n_estimators': 20000},
                                    cost_parameter_min={'n_estimators': 2000},
                                    n_jobs=4,
                                    cv=2)

from civismlext.stacking import StackedClassifier
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier


X, y = ... 
estimator_list = [('rf', RandomForestClassifier()), ('meta', LogisticRegression())]
cv = RandomizedSearchCV(
    estimator=StackedClassifier(
        estimator_list=estimator_list
    ),
    param_distributions={
        'rf__n_estimators': [10, 100, 1000, 10000],
        'rf__max_features': [None, 'sqrt', 'auto', 'log2'],
        'rf__criterion': ['gini', 'entropy'],
        'rf__class_weight': ['balanced_subsample', 'balanced'],
        'meta__l1_ratio': np.logspace(-5, 0, 6),
        'meta__C': np.logspace(-5, 5, 11),
    },
    scoring='roc_auc',
    n_iter=10
)
cv.fit(X, y)