GraphNN Classification¶
In [1]:
Copied!
from photonai.base import Hyperpipe, PipelineElement
from photonai_graph.GraphUtilities import get_random_connectivity_data, get_random_labels
from sklearn.model_selection import KFold
from photonai.base import Hyperpipe, PipelineElement
from photonai_graph.GraphUtilities import get_random_connectivity_data, get_random_labels
from sklearn.model_selection import KFold
Make random matrices to simulate connectivity matrices
In [2]:
Copied!
X = get_random_connectivity_data(number_of_nodes=50, number_of_individuals=100)
y = get_random_labels(l_type="classification", number_of_labels=100)
X = get_random_connectivity_data(number_of_nodes=50, number_of_individuals=100)
y = get_random_labels(l_type="classification", number_of_labels=100)
Design your Pipeline We add a simple GraphConstructor and a GCN Classifier
In [3]:
Copied!
my_pipe = Hyperpipe('basic_gembedding_pipe',
inner_cv=KFold(n_splits=5),
outer_cv=KFold(n_splits=5),
optimizer='sk_opt',
optimizer_params={'n_configurations': 25},
metrics=['accuracy', 'balanced_accuracy', 'recall', 'precision'],
best_config_metric='mean_absolute_error')
my_pipe.add(PipelineElement('GraphConstructorThreshold', threshold=0.95))
my_pipe.add(PipelineElement('GCNClassifier'))
my_pipe = Hyperpipe('basic_gembedding_pipe',
inner_cv=KFold(n_splits=5),
outer_cv=KFold(n_splits=5),
optimizer='sk_opt',
optimizer_params={'n_configurations': 25},
metrics=['accuracy', 'balanced_accuracy', 'recall', 'precision'],
best_config_metric='mean_absolute_error')
my_pipe.add(PipelineElement('GraphConstructorThreshold', threshold=0.95))
my_pipe.add(PipelineElement('GCNClassifier'))
Finally we simply fit the hyperpipe to our data.
In [4]:
Copied!
my_pipe.fit(X, y)
my_pipe.fit(X, y)
===================================================================================================== PHOTONAI ANALYSIS: basic_gembedding_pipe ===================================================================================================== Found 2 target classes: [0. 1.] ***************************************************************************************************** Outer Cross validation Fold 1 ***************************************************************************************************** Did not find any hyperparameter to convert into skopt space.
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/photonai/optimization/scikit_optimize/sk_opt.py:169: UserWarning: Did not find any hyperparameter to convert into skopt space. warnings.warn(msg) 100%|██████████| 200/200 [00:05<00:00, 37.45it/s] 100%|██████████| 200/200 [00:05<00:00, 36.98it/s] 100%|██████████| 200/200 [00:05<00:00, 38.35it/s] 100%|██████████| 200/200 [00:05<00:00, 37.37it/s] 100%|██████████| 200/200 [00:05<00:00, 39.01it/s]
-----------------------------------------------------------------------------------------------------
BEST_CONFIG
-----------------------------------------------------------------------------------------------------
{}
-----------------------------------------------------------------------------------------------------
VALIDATION PERFORMANCE
-----------------------------------------------------------------------------------------------------
+---------------------+-------------------+------------------+
| METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST |
+---------------------+-------------------+------------------+
| accuracy | 0.5750 | 0.5000 |
| balanced_accuracy | 0.5539 | 0.5170 |
| recall | 0.8220 | 0.7911 |
| precision | 0.5846 | 0.5446 |
| mean_absolute_error | 0.4250 | 0.5000 |
+---------------------+-------------------+------------------+
100%|██████████| 200/200 [00:06<00:00, 28.93it/s]
----------------------------------------------------------------------------------------------------- TEST PERFORMANCE ----------------------------------------------------------------------------------------------------- +---------------------+-------------------+------------------+ | METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST | +---------------------+-------------------+------------------+ | accuracy | 0.5875 | 0.3500 | | balanced_accuracy | 0.5921 | 0.3500 | | recall | 0.5000 | 0.1000 | | precision | 0.6364 | 0.2000 | | mean_absolute_error | 0.4125 | 0.6500 | +---------------------+-------------------+------------------+ ***************************************************************************************************** Outer Cross validation Fold 2 ***************************************************************************************************** Did not find any hyperparameter to convert into skopt space.
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/photonai/optimization/scikit_optimize/sk_opt.py:169: UserWarning: Did not find any hyperparameter to convert into skopt space. warnings.warn(msg) 100%|██████████| 200/200 [00:05<00:00, 36.69it/s] 100%|██████████| 200/200 [00:05<00:00, 36.49it/s] 100%|██████████| 200/200 [00:05<00:00, 37.70it/s] 100%|██████████| 200/200 [00:05<00:00, 37.04it/s] /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result)) /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result)) 100%|██████████| 200/200 [00:05<00:00, 36.32it/s] /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
-----------------------------------------------------------------------------------------------------
BEST_CONFIG
-----------------------------------------------------------------------------------------------------
{}
-----------------------------------------------------------------------------------------------------
VALIDATION PERFORMANCE
-----------------------------------------------------------------------------------------------------
+---------------------+-------------------+------------------+
| METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST |
+---------------------+-------------------+------------------+
| accuracy | 0.5469 | 0.4750 |
| balanced_accuracy | 0.5406 | 0.5000 |
| recall | 0.5312 | 0.4250 |
| precision | 0.3452 | 0.2752 |
| mean_absolute_error | 0.4531 | 0.5250 |
+---------------------+-------------------+------------------+
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result)) 100%|██████████| 200/200 [00:07<00:00, 27.04it/s]
----------------------------------------------------------------------------------------------------- TEST PERFORMANCE
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result))
----------------------------------------------------------------------------------------------------- +---------------------+-------------------+------------------+ | METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST | +---------------------+-------------------+------------------+ | accuracy | 0.5375 | 0.4000 | | balanced_accuracy | 0.5375 | 0.5000 | | recall | 0.1500 | 0.0000 | | precision | 0.6667 | 0.0000 | | mean_absolute_error | 0.4625 | 0.6000 | +---------------------+-------------------+------------------+ ***************************************************************************************************** Outer Cross validation Fold 3 ***************************************************************************************************** Did not find any hyperparameter to convert into skopt space.
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/photonai/optimization/scikit_optimize/sk_opt.py:169: UserWarning: Did not find any hyperparameter to convert into skopt space. warnings.warn(msg) 100%|██████████| 200/200 [00:05<00:00, 35.95it/s] 100%|██████████| 200/200 [00:05<00:00, 36.70it/s] 100%|██████████| 200/200 [00:05<00:00, 36.58it/s] 100%|██████████| 200/200 [00:05<00:00, 36.65it/s] 100%|██████████| 200/200 [00:05<00:00, 37.62it/s]
-----------------------------------------------------------------------------------------------------
BEST_CONFIG
-----------------------------------------------------------------------------------------------------
{}
-----------------------------------------------------------------------------------------------------
VALIDATION PERFORMANCE
-----------------------------------------------------------------------------------------------------
+---------------------+-------------------+------------------+
| METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST |
+---------------------+-------------------+------------------+
| accuracy | 0.5375 | 0.5375 |
| balanced_accuracy | 0.5000 | 0.5000 |
| recall | 1.0000 | 1.0000 |
| precision | 0.5375 | 0.5375 |
| mean_absolute_error | 0.4625 | 0.4625 |
+---------------------+-------------------+------------------+
100%|██████████| 200/200 [00:07<00:00, 28.07it/s]
----------------------------------------------------------------------------------------------------- TEST PERFORMANCE ----------------------------------------------------------------------------------------------------- +---------------------+-------------------+------------------+ | METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST | +---------------------+-------------------+------------------+ | accuracy | 0.5375 | 0.4500 | | balanced_accuracy | 0.5000 | 0.5000 | | recall | 1.0000 | 1.0000 | | precision | 0.5375 | 0.4500 | | mean_absolute_error | 0.4625 | 0.5500 | +---------------------+-------------------+------------------+ ***************************************************************************************************** Outer Cross validation Fold 4 ***************************************************************************************************** Did not find any hyperparameter to convert into skopt space.
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/photonai/optimization/scikit_optimize/sk_opt.py:169: UserWarning: Did not find any hyperparameter to convert into skopt space. warnings.warn(msg) 100%|██████████| 200/200 [00:05<00:00, 36.14it/s] 100%|██████████| 200/200 [00:05<00:00, 36.39it/s] 100%|██████████| 200/200 [00:05<00:00, 36.75it/s] /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1334: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior. _warn_prf(average, modifier, msg_start, len(result)) 100%|██████████| 200/200 [00:05<00:00, 35.87it/s] 100%|██████████| 200/200 [00:05<00:00, 37.28it/s]
-----------------------------------------------------------------------------------------------------
BEST_CONFIG
-----------------------------------------------------------------------------------------------------
{}
-----------------------------------------------------------------------------------------------------
VALIDATION PERFORMANCE
-----------------------------------------------------------------------------------------------------
+---------------------+-------------------+------------------+
| METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST |
+---------------------+-------------------+------------------+
| accuracy | 0.5219 | 0.4000 |
| balanced_accuracy | 0.5106 | 0.4325 |
| recall | 0.6384 | 0.5222 |
| precision | 0.5314 | 0.3208 |
| mean_absolute_error | 0.4781 | 0.6000 |
+---------------------+-------------------+------------------+
100%|██████████| 200/200 [00:07<00:00, 28.12it/s]
----------------------------------------------------------------------------------------------------- TEST PERFORMANCE ----------------------------------------------------------------------------------------------------- +---------------------+-------------------+------------------+ | METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST | +---------------------+-------------------+------------------+ | accuracy | 0.5125 | 0.5500 | | balanced_accuracy | 0.5000 | 0.5000 | | recall | 1.0000 | 1.0000 | | precision | 0.5125 | 0.5500 | | mean_absolute_error | 0.4875 | 0.4500 | +---------------------+-------------------+------------------+ ***************************************************************************************************** Outer Cross validation Fold 5 ***************************************************************************************************** Did not find any hyperparameter to convert into skopt space.
/opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/photonai/optimization/scikit_optimize/sk_opt.py:169: UserWarning: Did not find any hyperparameter to convert into skopt space. warnings.warn(msg) 100%|██████████| 200/200 [00:05<00:00, 36.90it/s] 100%|██████████| 200/200 [00:05<00:00, 35.18it/s] 100%|██████████| 200/200 [00:05<00:00, 37.23it/s] 100%|██████████| 200/200 [00:05<00:00, 36.31it/s] 100%|██████████| 200/200 [00:05<00:00, 36.55it/s]
-----------------------------------------------------------------------------------------------------
BEST_CONFIG
-----------------------------------------------------------------------------------------------------
{}
-----------------------------------------------------------------------------------------------------
VALIDATION PERFORMANCE
-----------------------------------------------------------------------------------------------------
+---------------------+-------------------+------------------+
| METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST |
+---------------------+-------------------+------------------+
| accuracy | 0.5250 | 0.5250 |
| balanced_accuracy | 0.5000 | 0.5000 |
| recall | 1.0000 | 1.0000 |
| precision | 0.5250 | 0.5250 |
| mean_absolute_error | 0.4750 | 0.4750 |
+---------------------+-------------------+------------------+
100%|██████████| 200/200 [00:07<00:00, 28.26it/s]
----------------------------------------------------------------------------------------------------- TEST PERFORMANCE ----------------------------------------------------------------------------------------------------- +---------------------+-------------------+------------------+ | METRIC | PERFORMANCE TRAIN | PERFORMANCE TEST | +---------------------+-------------------+------------------+ | accuracy | 0.5250 | 0.5000 | | balanced_accuracy | 0.5000 | 0.5000 | | recall | 1.0000 | 1.0000 | | precision | 0.5250 | 0.5000 | | mean_absolute_error | 0.4750 | 0.5000 | +---------------------+-------------------+------------------+ ***************************************************************************************************** Finished all outer fold computations.
100%|██████████| 200/200 [00:09<00:00, 20.86it/s]
*****************************************************************************************************
ANALYSIS INFORMATION ================================================================================
Project Folder: /home/runner/work/photonai_graph/photonai_graph/documentation/docs/examples/basic_gembedding_pipe_results_2023-02-17_16-32-02,
Computation Time: 2023-02-17 16:32:02.883945 - 2023-02-17 16:34:59.411549
Duration: 0:02:56.527604
Optimized for: mean_absolute_error
Hyperparameter Optimizer: sk_opt
DUMMY RESULTS =======================================================================================
+-------------------+--+
| PERFORMANCE DUMMY | |
+-------------------+--+
+-------------------+--+
AVERAGE PERFORMANCE ACROSS OUTER FOLDS ==============================================================
+---------------------+---------------+--------------+-----------+----------+
| Metric Name | Training Mean | Training Std | Test Mean | Test Std |
+---------------------+---------------+--------------+-----------+----------+
| accuracy | 0.54 | 0.025495 | 0.45 | 0.070711 |
| balanced_accuracy | 0.525921 | 0.036139 | 0.47 | 0.06 |
| recall | 0.73 | 0.348712 | 0.62 | 0.466476 |
| precision | 0.575606 | 0.063212 | 0.34 | 0.208327 |
| mean_absolute_error | 0.46 | 0.025495 | 0.55 | 0.070711 |
+---------------------+---------------+--------------+-----------+----------+
BEST HYPERPARAMETER CONFIGURATION ===================================================================
{}
+--------+----------+-------------------+--------+-----------+---------------------+----------------------------+
| fold # | accuracy | balanced_accuracy | recall | precision | mean_absolute_error | Best Hyperparameter Config |
+--------+----------+-------------------+--------+-----------+---------------------+----------------------------+
| 1 | 0.3500 | 0.3500 | 0.1000 | 0.2000 | 0.6500 | {} |
| 2 | 0.4000 | 0.5000 | 0.0000 | 0.0000 | 0.6000 | {} |
| 3 | 0.4500 | 0.5000 | 1.0000 | 0.4500 | 0.5500 | {} |
| 4* | 0.5500 | 0.5000 | 1.0000 | 0.5500 | 0.4500 | {} |
| 5 | 0.5000 | 0.5000 | 1.0000 | 0.5000 | 0.5000 | {} |
+--------+----------+-------------------+--------+-----------+---------------------+----------------------------+
PHOTONAI 2.3.0 ======================================================================================
Your results are stored in /home/runner/work/photonai_graph/photonai_graph/documentation/docs/examples/basic_gembedding_pipe_results_2023-02-17_16-32-02
Go to https://explorer.photon-ai.com and upload your photonai_results.json for convenient result visualization!
For more info and documentation visit https://www.photon-ai.com
--------------------------------------------------------------------------- AttributeError Traceback (most recent call last) File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/IPython/core/formatters.py:972, in MimeBundleFormatter.__call__(self, obj, include, exclude) 969 method = get_real_method(obj, self.print_method) 971 if method is not None: --> 972 return method(include=include, exclude=exclude) 973 return None 974 else: File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/base.py:631, in BaseEstimator._repr_mimebundle_(self, **kwargs) 629 output = {"text/plain": repr(self)} 630 if get_config()["display"] == "diagram": --> 631 output["text/html"] = estimator_html_repr(self) 632 return output File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:417, in estimator_html_repr(estimator) 403 fallback_msg = ( 404 "In a Jupyter environment, please rerun this cell to show the HTML" 405 " representation or trust the notebook. <br />On GitHub, the" 406 " HTML representation is unable to render, please try loading this page" 407 " with nbviewer.org." 408 ) 409 out.write( 410 f"<style>{style_with_id}</style>" 411 f'<div id="{container_id}" class="sk-top-container">' (...) 415 '<div class="sk-container" hidden>' 416 ) --> 417 _write_estimator_html( 418 out, 419 estimator, 420 estimator.__class__.__name__, 421 estimator_str, 422 first_call=True, 423 ) 424 out.write("</div></div>") 426 html_output = out.getvalue() File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:151, in _write_estimator_html(out, estimator, estimator_label, estimator_label_details, first_call) 149 """Write estimator to html in serial, parallel, or by itself (single).""" 150 if first_call: --> 151 est_block = _get_visual_block(estimator) 152 else: 153 with config_context(print_changed_only=True): File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:127, in _get_visual_block(estimator) 123 # check if estimator looks like a meta estimator wraps estimators 124 if hasattr(estimator, "get_params"): 125 estimators = [ 126 (key, est) --> 127 for key, est in estimator.get_params(deep=False).items() 128 if hasattr(est, "get_params") and hasattr(est, "fit") 129 ] 130 if estimators: 131 return _VisualBlock( 132 "parallel", 133 [est for _, est in estimators], 134 names=[f"{key}: {est.__class__.__name__}" for key, est in estimators], 135 name_details=[str(est) for _, est in estimators], 136 ) File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/base.py:211, in BaseEstimator.get_params(self, deep) 209 out = dict() 210 for key in self._get_param_names(): --> 211 value = getattr(self, key) 212 if deep and hasattr(value, "get_params") and not isinstance(value, type): 213 deep_items = value.get_params().items() AttributeError: 'Hyperpipe' object has no attribute 'best_config_metric'
--------------------------------------------------------------------------- AttributeError Traceback (most recent call last) File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/IPython/core/formatters.py:342, in BaseFormatter.__call__(self, obj) 340 method = get_real_method(obj, self.print_method) 341 if method is not None: --> 342 return method() 343 return None 344 else: File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/base.py:625, in BaseEstimator._repr_html_inner(self) 620 def _repr_html_inner(self): 621 """This function is returned by the @property `_repr_html_` to make 622 `hasattr(estimator, "_repr_html_") return `True` or `False` depending 623 on `get_config()["display"]`. 624 """ --> 625 return estimator_html_repr(self) File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:417, in estimator_html_repr(estimator) 403 fallback_msg = ( 404 "In a Jupyter environment, please rerun this cell to show the HTML" 405 " representation or trust the notebook. <br />On GitHub, the" 406 " HTML representation is unable to render, please try loading this page" 407 " with nbviewer.org." 408 ) 409 out.write( 410 f"<style>{style_with_id}</style>" 411 f'<div id="{container_id}" class="sk-top-container">' (...) 415 '<div class="sk-container" hidden>' 416 ) --> 417 _write_estimator_html( 418 out, 419 estimator, 420 estimator.__class__.__name__, 421 estimator_str, 422 first_call=True, 423 ) 424 out.write("</div></div>") 426 html_output = out.getvalue() File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:151, in _write_estimator_html(out, estimator, estimator_label, estimator_label_details, first_call) 149 """Write estimator to html in serial, parallel, or by itself (single).""" 150 if first_call: --> 151 est_block = _get_visual_block(estimator) 152 else: 153 with config_context(print_changed_only=True): File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/utils/_estimator_html_repr.py:127, in _get_visual_block(estimator) 123 # check if estimator looks like a meta estimator wraps estimators 124 if hasattr(estimator, "get_params"): 125 estimators = [ 126 (key, est) --> 127 for key, est in estimator.get_params(deep=False).items() 128 if hasattr(est, "get_params") and hasattr(est, "fit") 129 ] 130 if estimators: 131 return _VisualBlock( 132 "parallel", 133 [est for _, est in estimators], 134 names=[f"{key}: {est.__class__.__name__}" for key, est in estimators], 135 name_details=[str(est) for _, est in estimators], 136 ) File /opt/hostedtoolcache/Python/3.9.16/x64/lib/python3.9/site-packages/sklearn/base.py:211, in BaseEstimator.get_params(self, deep) 209 out = dict() 210 for key in self._get_param_names(): --> 211 value = getattr(self, key) 212 if deep and hasattr(value, "get_params") and not isinstance(value, type): 213 deep_items = value.get_params().items() AttributeError: 'Hyperpipe' object has no attribute 'best_config_metric'
Out[4]:
Hyperpipe(name='basic_gembedding_pipe')