Skip to content

Commit

Permalink
format
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexanderDokuchaev
AlexanderDokuchaev committed Apr 17, 2023
1 parent e403fb8 commit 73cd999
Show file tree
Hide file tree
Showing 889 changed files with 42,096 additions and 38,741 deletions.
41 changes: 28 additions & 13 deletions examples/common/sample_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,18 @@
class ActionWrapper(argparse.Action):
def __init__(self, action):
self._action = action
super().__init__(action.option_strings, action.dest, nargs=action.nargs, const=action.const,
default=action.default, type=action.type, choices=action.choices, required=action.required,
help=action.help, metavar=action.metavar)
super().__init__(
action.option_strings,
action.dest,
nargs=action.nargs,
const=action.const,
default=action.default,
type=action.type,
choices=action.choices,
required=action.required,
help=action.help,
metavar=action.metavar,
)
self._action = action

def __getattr__(self, item):
Expand All @@ -41,13 +50,13 @@ def __call__(self, parser, namespace, values, option_string=None):
return self._action(parser, namespace, values, option_string)


#pylint:disable=protected-access
# pylint:disable=protected-access
class CustomArgumentGroup(argparse._ArgumentGroup):
def _add_action(self, action):
super()._add_action(ActionWrapper(action))


#pylint:disable=protected-access
# pylint:disable=protected-access
class CustomActionContainer(argparse._ActionsContainer):
def add_argument_group(self, *args, **kwargs):
group = CustomArgumentGroup(self, *args, **kwargs)
Expand All @@ -69,7 +78,7 @@ def parse_known_args(self, args=None, namespace=None):

class SampleConfig(Dict):
@classmethod
def from_json(cls, path) -> 'SampleConfig':
def from_json(cls, path) -> "SampleConfig":
file_path = Path(path).resolve()
with safe_open(file_path) as f:
loaded_json = json.load(f)
Expand Down Expand Up @@ -98,16 +107,20 @@ def update_from_env(self, key_to_env_dict=None):
self[k] = int(os.environ[v])


EVAL_ONLY_ERROR_TEXT = 'The config file you are using is only presented for purposes of running the model ' \
'in evaluation mode.\n If you wish to run training for this model, remove the ' \
'`"eval_only": true` line from the .json configuration file and provide training ' \
'hyperparameters (e.g. number of training epochs, optimizer etc.) in the same config.'
EVAL_ONLY_ERROR_TEXT = (
"The config file you are using is only presented for purposes of running the model "
"in evaluation mode.\n If you wish to run training for this model, remove the "
'`"eval_only": true` line from the .json configuration file and provide training '
"hyperparameters (e.g. number of training epochs, optimizer etc.) in the same config."
)


def _parse_sample_config(args, parser) -> SampleConfig:
sample_config = SampleConfig.from_json(args.config)
sample_config.update_from_args(args, parser)
return sample_config


def _embed_nncf_config(args, sample_config: SampleConfig) -> SampleConfig:
file_path = Path(args.config).resolve()
with safe_open(file_path) as f:
Expand All @@ -118,16 +131,18 @@ def _embed_nncf_config(args, sample_config: SampleConfig) -> SampleConfig:
loaded_json["target_device"] = target_device
nncf_config = NNCFConfig.from_dict(loaded_json)

if args.disable_compression and 'compression' in nncf_config:
del nncf_config['compression']
if args.disable_compression and "compression" in nncf_config:
del nncf_config["compression"]

sample_config.nncf_config = nncf_config
return sample_config


def _fail_if_training_with_eval_only_config(sample_config: SampleConfig):
if sample_config.eval_only and 'train' in sample_config.mode:
if sample_config.eval_only and "train" in sample_config.mode:
raise RuntimeError(EVAL_ONLY_ERROR_TEXT)


def create_sample_config(args, parser, **kwargs) -> SampleConfig:
sample_config = _parse_sample_config(args, parser)
sample_config.update(**kwargs)
Expand Down
85 changes: 41 additions & 44 deletions examples/experimental/openvino/bert/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,28 @@
limitations under the License.
"""

from typing import Iterable
from typing import Any

import subprocess
from pathlib import Path
from typing import Any, Iterable

import openvino.runtime as ov
import torch
import nncf
import datasets
import evaluate
import transformers
import numpy as np
import openvino.runtime as ov
import torch
import transformers

import nncf
from nncf.parameters import ModelType

# Path to the `bert` directory.
ROOT = Path(__file__).parent.resolve()
# Path to the directory where the original and quantized IR will be saved.
MODEL_DIR = ROOT / 'bert_quantization'
MODEL_DIR = ROOT / "bert_quantization"
# Path to the pre-trained model directory.
PRETRAINED_MODEL_DIR = ROOT / 'MRPC'
PRETRAINED_MODEL_DIR = ROOT / "MRPC"

TASK_NAME = 'mrpc'
TASK_NAME = "mrpc"
MAX_SEQ_LENGTH = 128


Expand All @@ -61,30 +59,28 @@ def run_example():
# per iteration through the `data_source` object and transform it into the model's
# expected input that can be used for the model inference.
INPUT_NAMES = [x.any_name for x in ov_model.inputs]

def transform_fn(data_item):
inputs = {
name: np.asarray(data_item[name], dtype=np.int64) for name in INPUT_NAMES
}
inputs = {name: np.asarray(data_item[name], dtype=np.int64) for name in INPUT_NAMES}
return [inputs]

# Wrap framework-specific data source into the `nncf.Dataset` object.
calibration_dataset = nncf.Dataset(data_source, transform_fn)
quantized_model = nncf.quantize(ov_model, calibration_dataset,
model_type=ModelType.TRANSFORMER)
quantized_model = nncf.quantize(ov_model, calibration_dataset, model_type=ModelType.TRANSFORMER)

# Step 5: Save quantized model.
ir_qmodel_xml = MODEL_DIR / 'bert_base_mrpc_quantized.xml'
ir_qmodel_bin = MODEL_DIR / 'bert_base_mrpc_quantized.bin'
ir_qmodel_xml = MODEL_DIR / "bert_base_mrpc_quantized.xml"
ir_qmodel_bin = MODEL_DIR / "bert_base_mrpc_quantized.bin"
ov.serialize(quantized_model, str(ir_qmodel_xml), str(ir_qmodel_bin))

# Step 6: Compare the accuracy of the original and quantized models.
print('Checking the accuracy of the original model:')
print("Checking the accuracy of the original model:")
metric = validation_fn(ov_model, data_source)
print(f'F1 score: {metric}')
print(f"F1 score: {metric}")

print('Checking the accuracy of the quantized model:')
print("Checking the accuracy of the quantized model:")
metric = validation_fn(quantized_model, data_source)
print(f'F1 score: {metric}')
print(f"F1 score: {metric}")

# Step 7: Compare Performance of the original and quantized models.
# benchmark_app -m bert_quantization/bert_base_mrpc.xml -d CPU -api async
Expand All @@ -99,26 +95,28 @@ def convert_torch_to_openvino(model: torch.nn.Module) -> ov.Model:
MODEL_DIR.mkdir()

# Export PyTorch model to the ONNX format.
onnx_model_path = MODEL_DIR / 'bert_base_mrpc.onnx'
onnx_model_path = MODEL_DIR / "bert_base_mrpc.onnx"
dummy_input = (
torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64), # input_ids
torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64), # attention_mask
torch.ones(1, MAX_SEQ_LENGTH, dtype=torch.int64), # token_type_ids
)
torch.onnx.export(model,
dummy_input,
onnx_model_path,
verbose=False,
opset_version=11,
input_names=['input_ids', 'attention_mask', 'token_type_ids'],
output_names=['output'])
torch.onnx.export(
model,
dummy_input,
onnx_model_path,
verbose=False,
opset_version=11,
input_names=["input_ids", "attention_mask", "token_type_ids"],
output_names=["output"],
)

# Run Model Optimizer to convert ONNX model to OpenVINO IR.
mo_command = f'mo --framework onnx -m {onnx_model_path} --output_dir {MODEL_DIR}'
mo_command = f"mo --framework onnx -m {onnx_model_path} --output_dir {MODEL_DIR}"
subprocess.call(mo_command, shell=True) # nosec

ir_model_xml = MODEL_DIR / 'bert_base_mrpc.xml'
ir_model_bin = MODEL_DIR / 'bert_base_mrpc.bin'
ir_model_xml = MODEL_DIR / "bert_base_mrpc.xml"
ir_model_bin = MODEL_DIR / "bert_base_mrpc.bin"
ov_model = ie.read_model(model=ir_model_xml, weights=ir_model_bin)

return ov_model
Expand All @@ -130,37 +128,36 @@ def create_data_source() -> datasets.Dataset:
:return: The `datasets.Dataset` object.
"""
raw_dataset = datasets.load_dataset('glue', TASK_NAME, split='validation')
raw_dataset = datasets.load_dataset("glue", TASK_NAME, split="validation")
tokenizer = transformers.BertTokenizer.from_pretrained(PRETRAINED_MODEL_DIR)

def _preprocess_fn(examples):
texts = (examples['sentence1'], examples['sentence2'])
result = tokenizer(*texts, padding='max_length', max_length=MAX_SEQ_LENGTH, truncation=True)
result['labels'] = examples['label']
texts = (examples["sentence1"], examples["sentence2"])
result = tokenizer(*texts, padding="max_length", max_length=MAX_SEQ_LENGTH, truncation=True)
result["labels"] = examples["label"]
return result

processed_dataset = raw_dataset.map(_preprocess_fn, batched=True, batch_size=1)

return processed_dataset


def validation_fn(model: ov.Model, validation_dataset: Iterable[Any]) -> float:
compiled_model = ie.compile_model(model, device_name='CPU')
compiled_model = ie.compile_model(model, device_name="CPU")
output_layer = compiled_model.output(0)

metric = evaluate.load('glue', TASK_NAME)
metric = evaluate.load("glue", TASK_NAME)
INPUT_NAMES = [x.any_name for x in compiled_model.inputs]
for batch in validation_dataset:
inputs = [
np.expand_dims(np.asarray(batch[key], dtype=np.int64), 0) for key in INPUT_NAMES
]
inputs = [np.expand_dims(np.asarray(batch[key], dtype=np.int64), 0) for key in INPUT_NAMES]
outputs = compiled_model(inputs)[output_layer]
predictions = outputs[0].argmax(axis=-1)
metric.add_batch(predictions=[predictions], references=[batch['labels']])
metric.add_batch(predictions=[predictions], references=[batch["labels"]])
metrics = metric.compute()
f1_score = metrics['f1']
f1_score = metrics["f1"]

return f1_score


if __name__ == '__main__':
if __name__ == "__main__":
run_example()
54 changes: 27 additions & 27 deletions examples/experimental/openvino/ssd_mobilenet_v1_fpn/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,25 @@
limitations under the License.
"""

from typing import Iterable
from typing import Any
from pathlib import Path
from functools import partial
from pathlib import Path
from typing import Any, Iterable

import nncf
import numpy as np
import openvino.runtime as ov

from openvino.tools.pot.api.samples.object_detection.data_loader import COCOLoader
from openvino.tools.pot.api.samples.object_detection.metric import MAP

import nncf

FILE = Path(__file__).resolve()
# Relative path to the `ssd_mobilenet_v1_fpn` directory.
ROOT = FILE.parent.relative_to(Path.cwd())
# Path to the directory where the original and quantized IR will be saved.
MODEL_DIR = ROOT / 'ssd_mobilenet_v1_fpn_quantization'
MODEL_DIR = ROOT / "ssd_mobilenet_v1_fpn_quantization"
# Path to COCO validation dataset.
DATASET_DIR = ROOT / 'coco2017' / 'val2017'
ANNOTATION_FILE = ROOT / 'coco2017' / 'instances_val2017.json'
DATASET_DIR = ROOT / "coco2017" / "val2017"
ANNOTATION_FILE = ROOT / "coco2017" / "instances_val2017.json"


ie = ov.Core()
Expand All @@ -42,13 +40,13 @@ def run_example():
Runs the SSD MobileNetV1 FPN quantize with accuracy control example.
"""
# Step 1: Load the OpenVINO model.
ir_model_xml = ROOT / 'public' / 'ssd_mobilenet_v1_fpn_coco' / 'FP32' / 'ssd_mobilenet_v1_fpn_coco.xml'
ir_model_xml = ROOT / "public" / "ssd_mobilenet_v1_fpn_coco" / "FP32" / "ssd_mobilenet_v1_fpn_coco.xml"
ov_model = ie.read_model(ir_model_xml)

# Step 2: Create data source.
dataset_config = {
'images_path': f'{DATASET_DIR}/',
'annotation_path': ANNOTATION_FILE,
"images_path": f"{DATASET_DIR}/",
"annotation_path": ANNOTATION_FILE,
}
data_source = COCOLoader(dataset_config)

Expand All @@ -64,31 +62,33 @@ def transform_fn(data):
metric = MAP(91, data_source.labels)
# Wrap framework-specific data source into the `nncf.Dataset` object.
validation_dataset = nncf.Dataset(data_source, transform_fn)
quantized_model = nncf.quantize_with_accuracy_control(ov_model,
validation_dataset,
validation_dataset,
validation_fn=partial(validation_fn, metric=metric),
max_drop=0.004,
preset=nncf.QuantizationPreset.MIXED)
quantized_model = nncf.quantize_with_accuracy_control(
ov_model,
validation_dataset,
validation_dataset,
validation_fn=partial(validation_fn, metric=metric),
max_drop=0.004,
preset=nncf.QuantizationPreset.MIXED,
)

# Step 4: Save the quantized model.
if not MODEL_DIR.exists():
MODEL_DIR.mkdir()

ir_qmodel_xml = MODEL_DIR / 'ssd_mobilenet_v1_fpn_quantized.xml'
ir_qmodel_bin = MODEL_DIR / 'ssd_mobilenet_v1_fpn_quantized.bin'
ir_qmodel_xml = MODEL_DIR / "ssd_mobilenet_v1_fpn_quantized.xml"
ir_qmodel_bin = MODEL_DIR / "ssd_mobilenet_v1_fpn_quantized.bin"
ov.serialize(quantized_model, str(ir_qmodel_xml), str(ir_qmodel_bin))

# Step 6: Compare the accuracy of the original and quantized models.
print('Checking the accuracy of the original model:')
compiled_model = ie.compile_model(ov_model, device_name='CPU')
print("Checking the accuracy of the original model:")
compiled_model = ie.compile_model(ov_model, device_name="CPU")
metric = validation_fn(compiled_model, data_source, data_source.labels)
print(f'mAP: {metric}')
print(f"mAP: {metric}")

print('Checking the accuracy of the quantized model:')
compiled_model = ie.compile_model(quantized_model, device_name='CPU')
print("Checking the accuracy of the quantized model:")
compiled_model = ie.compile_model(quantized_model, device_name="CPU")
metric = validation_fn(compiled_model, data_source, data_source.labels)
print(f'mAP: {metric}')
print(f"mAP: {metric}")

# Step 7: Compare Performance of the original and quantized models.
# benchmark_app -m ssd_mobilenet_v1_fpn_quantization/ssd_mobilenet_v1_fpn.xml -d CPU -api async
Expand All @@ -103,8 +103,8 @@ def validation_fn(compiled_model: ov.CompiledModel, validation_dataset: Iterable
output = compiled_model([input_data])[output_layer]
metric.update(output, [labels])

return metric.avg_value['MAP'].item()
return metric.avg_value["MAP"].item()


if __name__ == '__main__':
if __name__ == "__main__":
run_example()
Loading

0 comments on commit 73cd999

Please sign in to comment.