-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathutils.py
172 lines (145 loc) · 6.3 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
import os
import time
from torch.utils.tensorboard import SummaryWriter
import logging
def setattr_cls_from_kwargs(cls, kwargs):
#if default values are in the cls,
#overlap the value by kwargs
for key in kwargs.keys():
if hasattr(cls, key):
print(f"{key} in {cls} is overlapped by kwargs: {getattr(cls,key)} -> {kwargs[key]}")
setattr(cls, key, kwargs[key])
def test_setattr_cls_from_kwargs():
class _test_cls:
def __init__(self):
self.a = 1
self.b = 'hello'
test_cls = _test_cls()
config = {'a': 3, 'b': 'change_hello', 'c':5}
setattr_cls_from_kwargs(test_cls, config)
for key in config.keys():
print(f"{key}:\t {getattr(test_cls, key)}")
def net_builder(net_name, from_name: bool, net_conf=None):
"""
return **class** of backbone network (not instance).
Args
net_name: 'WideResNet' or network names in torchvision.models
from_name: If True, net_buidler takes models in torch.vision models. Then, net_conf is ignored.
net_conf: When from_name is False, net_conf is the configuration of backbone network (now, only WRN is supported).
"""
if from_name:
import torchvision.models as models
model_name_list = sorted(name for name in models.__dict__
if name.islower() and not name.startswith("__")
and callable(models.__dict__[name]))
if net_name not in model_name_list:
assert Exception(f"[!] Networks\' Name is wrong, check net config, \
expected: {model_name_list} \
received: {net_name}")
else:
return models.__dict__[net_name]
else:
if net_name == 'WideResNet':
import models.nets.wrn as net
builder = getattr(net, 'build_WideResNet')()
setattr_cls_from_kwargs(builder, net_conf)
elif net_name == 'ResNet':
import models.nets.resnet as net
builder = getattr(net, 'build_ResNet')()
return builder.build
def test_net_builder(net_name, from_name, net_conf=None):
builder = net_builder(net_name, from_name, net_conf)
print(f"net_name: {net_name}, from_name: {from_name}, net_conf: {net_conf}")
print(builder)
def get_logger(name, save_path=None, level='INFO'):
logger = logging.getLogger(name)
logger.setLevel(getattr(logging, level))
log_format = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s')
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(log_format)
logger.addHandler(streamHandler)
if not save_path is None:
os.makedirs(save_path, exist_ok=True)
fileHandler = logging.FileHandler(os.path.join(save_path, 'log.txt'))
fileHandler.setFormatter(log_format)
logger.addHandler(fileHandler)
return logger
def count_parameters(model):
return sum(p.numel() for p in model.parameters() if p.requires_grad)
import numpy as np
from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score, confusion_matrix
def get_metrics(outputs, labels, classes, tag="test/"):
'''
returns a dictionary of computed metrics
ARGS
outputs: (np.ndarray) a (N, # classes) dimensional array of output logits of the model
labels: (np.ndarray) a (N) dimensional array where each element is the ground truth
index of the corresponding output element
classes: (list) a list of stings of names of classes
RETURNS:
a dictionary of classification metircs, support for:
1. precision,
2. recall,
3. accuracy,
4. max precision across all classes
5. mean precision across all classes
6. min precision across all classes
7. max recall across all classes
8. mean recall across all classes
9. min recall across all classes
10. f1 micro average
11. f1 macroa average
12. Head recall
13. Tail recall
14. Head Coverage
15. Tail Coverage
'''
num_classes = len(classes)
precision = precision_score(labels, outputs, average=None, zero_division=0)
precision_avg = precision_score(labels, outputs, average='macro', zero_division=0)
max_precision = np.max(precision)
min_precision = np.min(precision)
mean_precision = np.mean(precision)
recall = recall_score(labels, outputs, average=None, zero_division=0)
tail_recall = np.mean(recall[int(0.9*num_classes):])
head_recall = np.mean(recall[:int(0.9*num_classes)])
minHT = min(tail_recall, head_recall)
recall_avg = recall_score(labels, outputs, average='macro', zero_division=0)
max_recall = np.max(recall)
min_recall = np.min(recall)
mean_recall = np.mean(recall)
f1_micro = f1_score(labels, outputs, average='micro')
f1_macro = f1_score(labels, outputs, average='macro')
CM = confusion_matrix(labels, outputs, normalize="all")
coverages = np.sum(CM, axis=0)
head_coverage, tail_coverage = np.mean(coverages[:int(0.9*num_classes)]), \
np.mean(coverages[int(0.9*num_classes):])
accuracy = accuracy_score(labels, outputs)
metrics = {
"precision": precision_avg,
"recall": recall_avg,
"accuracy": accuracy,
"max_precision": max_precision,
"mean_precision": mean_precision,
"min_precision": min_precision,
"max_recall": max_recall,
"mean_recall": mean_recall,
"min_recall": min_recall,
"f1_micro": f1_micro,
"f1_macro": f1_macro,
"tail_recall": tail_recall,
"head_recall": head_recall,
"min_head_tail": minHT,
"head_coverage": head_coverage,
"tail_coverage": tail_coverage
}
for i, name in enumerate(classes):
metrics["precision_" + name] = precision[i]
metrics["recall_" + name] = recall[i]
for i, name in enumerate(classes):
metrics["coverage_" + name] = coverages[i]
metrics["min_coverage"] = min(coverages)
metrics_ = {}
for key in metrics.keys():
metrics_[tag + key] = metrics[key]
return metrics