Skip to content

Permutation Importance

PHOTONAI conveniently integrates scikit-learns permutation importance function to get the permutation feature importances for the optimum pipe after optimization is finished.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
from sklearn.datasets import load_diabetes
from sklearn.model_selection import KFold, train_test_split

from photonai.base import Hyperpipe, PipelineElement

diabetes = load_diabetes()
X_train, X_val, y_train, y_val = train_test_split(diabetes.data, diabetes.target, random_state=0)

my_pipe = Hyperpipe('basic_ridge_pipe',
                    inner_cv=KFold(n_splits=5),
                    outer_cv=KFold(n_splits=3),
                    optimizer='grid_search',
                    metrics=['mean_absolute_error'],
                    best_config_metric='mean_absolute_error',
                    project_folder='./tmp')

my_pipe += PipelineElement("StandardScaler")
my_pipe += PipelineElement('Ridge', alpha=1e-2)
my_pipe.fit(X_train, y_train)

r = my_pipe.get_permutation_feature_importances(n_repeats=50, random_state=0)

for i in r["mean"].argsort()[::-1]:
    if r["mean"][i] - 2 * r["std"][i] > 0:
        print(f"{diabetes.feature_names[i]:<8}"
              f"{r['mean'][i]:.3f}"
              f" +/- {r['std'][i]:.3f}")


# get permutation importances posthoc
# reloaded_hyperpipe = Hyperpipe.reload_hyperpipe("full_path/to/results_folder/", X_train, y_train)
# post_hoc_perm_importances = Hyperpipe.get_permutation_feature_importances(n_repeats=5, random_state=0)