-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathCLIP-L_sthv2_u32.py
238 lines (238 loc) · 7.76 KB
/
CLIP-L_sthv2_u32.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
default_scope = 'mmaction'
default_hooks = dict(
runtime_info=dict(type='RuntimeInfoHook'),
timer=dict(type='IterTimerHook'),
logger=dict(type='LoggerHook', interval=20, ignore_last=False),
param_scheduler=dict(type='ParamSchedulerHook'),
checkpoint=dict(
type='CheckpointHook', interval=1, max_keep_ckpts=5, save_best='auto'),
sampler_seed=dict(type='DistSamplerSeedHook'),
sync_buffers=dict(type='SyncBuffersHook'))
env_cfg = dict(
cudnn_benchmark=False,
mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0),
dist_cfg=dict(backend='nccl'))
log_processor = dict(type='LogProcessor', window_size=20, by_epoch=True)
vis_backends = [dict(type='LocalVisBackend')]
visualizer = dict(
type='ActionVisualizer', vis_backends=[dict(type='LocalVisBackend')])
log_level = 'INFO'
load_from = None
resume = True
model = dict(
type='Recognizer3D',
backbone=dict(
type='ViT_Zero_CLIP_long',
pretrained='your_clip_path',
input_resolution=224,
patch_size=14,
num_frames=32,
width=1024,
layers=24,
heads=16,
adapter_scale=1,
num_tadapter=2,
stdha_cfg=dict(
divide_head=False,
shift1_div=8,
shift2_div=16,
shift3_div=16,
shift3_right=False,
shift4_div=-1,
shift4_right=False)),
cls_head=dict(
type='I3DHead',
in_channels=1024,
num_classes=174,
spatial_type='avg',
dropout_ratio=0.5,
label_smooth_eps=0.1,
average_clips='prob'),
data_preprocessor=dict(
type='ActionDataPreprocessor',
mean=[122.769, 116.74, 104.04],
std=[68.493, 66.63, 70.321],
format_shape='NCTHW'))
dataset_type = 'VideoDataset'
data_prefix = 'your_path_sthv2/'
data_root = 'your_path_sthv2/videos'
data_root_val = 'your_path_sthv2/videos'
ann_file_train = 'data/sthv2/sthv2_train_list_videos.txt'
ann_file_val = 'data/sthv2/sthv2_val_list_videos.txt'
ann_file_test = 'data/sthv2/sthv2_val_list_videos.txt'
file_client_args = dict(io_backend='ceph')
sthv2_flip_label_map = dict({
86: 87,
87: 86,
93: 94,
94: 93,
166: 167,
167: 166
})
train_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='RandomResizedCrop'),
dict(type='Resize', scale=(224, 224), keep_ratio=False),
dict(
type='Flip',
flip_ratio=0.5,
flip_label_map=dict({
86: 87,
87: 86,
93: 94,
94: 93,
166: 167,
167: 166
})),
dict(
type='PytorchVideoWrapper',
op='RandAugment',
magnitude=7,
num_layers=4),
dict(type='RandomErasing', erase_prob=0.25),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
val_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32, test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='CenterCrop', crop_size=224),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
test_pipeline = [
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32, test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 224)),
dict(type='ThreeCrop', crop_size=224),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]
train_dataloader = dict(
batch_size=2,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=True),
dataset=dict(
type='VideoDataset',
ann_file='data/sthv2/sthv2_train_list_videos.txt',
data_prefix=dict(video='your_path_sthv2/videos'),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='RandomResizedCrop'),
dict(type='Resize', scale=(224, 224), keep_ratio=False),
dict(
type='Flip',
flip_ratio=0.5,
flip_label_map=dict({
86: 87,
87: 86,
93: 94,
94: 93,
166: 167,
167: 166
})),
dict(
type='PytorchVideoWrapper',
op='RandAugment',
magnitude=7,
num_layers=4),
dict(type='RandomErasing', erase_prob=0.25),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
]))
val_dataloader = dict(
batch_size=2,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False),
dataset=dict(
type='VideoDataset',
ann_file='data/sthv2/sthv2_val_list_videos.txt',
data_prefix=dict(video='your_path_sthv2/videos'),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32, test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 256)),
dict(type='CenterCrop', crop_size=224),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
],
test_mode=True))
test_dataloader = dict(
batch_size=1,
num_workers=8,
persistent_workers=True,
sampler=dict(type='DefaultSampler', shuffle=False),
dataset=dict(
type='VideoDataset',
ann_file='data/sthv2/sthv2_val_list_videos.txt',
data_prefix=dict(video='your_path_sthv2/videos'),
pipeline=[
dict(type='DecordInit', io_backend='ceph'),
dict(type='UniformSample', clip_len=32, test_mode=True),
dict(type='DecordDecode'),
dict(type='Resize', scale=(-1, 224)),
dict(type='ThreeCrop', crop_size=224),
dict(type='FormatShape', input_format='NCTHW'),
dict(type='PackActionInputs')
],
test_mode=True))
val_evaluator = dict(type='AccMetric')
test_evaluator = dict(type='AccMetric')
train_cfg = dict(
type='EpochBasedTrainLoop',
max_epochs=50,
val_begin=1,
dynamic_intervals=[(1, 5), (40, 1)])
val_cfg = dict(type='ValLoop')
test_cfg = dict(type='TestLoop')
accumulative_counts = 2
auto_scale_lr = dict(enable=True, base_batch_size=32)
optim_wrapper = dict(
type='GradMonitorAmpOptimWrapper',
optimizer=dict(
type='AdamW', lr=0.0003, betas=(0.9, 0.999), weight_decay=0.05),
constructor='GradMonitorSwinOptimWrapperConstructor',
accumulative_counts=2,
paramwise_cfg=dict(
class_embedding=dict(decay_mult=0.0),
positional_embedding=dict(decay_mult=0.0),
temporal_embedding=dict(decay_mult=0.0),
absolute_pos_embed=dict(decay_mult=0.0),
ln_1=dict(decay_mult=0.0),
ln_2=dict(decay_mult=0.0),
ln_pre=dict(decay_mult=0.0),
ln_post=dict(decay_mult=0.0),
backbone=dict(lr_mult=0.1)))
param_scheduler = [
dict(
type='LinearLR',
start_factor=0.1,
by_epoch=True,
begin=0,
end=2.5,
convert_to_iter_based=True),
dict(
type='CosineAnnealingLR',
T_max=50,
eta_min=0,
by_epoch=True,
begin=0,
end=50)
]
find_unused_parameters = True
launcher = 'slurm'
work_dir = '.work_dirs/recognition/vit_zero_clip/FinalS_vit_zero_large_clip_sthv2_u32_adapter_2'
randomness = dict(seed=None, diff_rank_seed=False, deterministic=False)
dist_params = dict(port=18757)