diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 24515f8..1686afe 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,13 +16,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] dependencies: [minimal, latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -53,9 +53,9 @@ jobs: if: ${{ github.event_name == 'push' }} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.7 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.7 - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae802f4..c53192e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,13 +17,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] dependencies: [minimal, latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -53,9 +53,9 @@ jobs: needs: [ build ] runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.7 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.7 - name: Install dependencies diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 5707030..86e8f20 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -17,13 +17,13 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12"] dependencies: [minimal, latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/cli.py b/cli.py index b7dc56f..93c40ec 100755 --- a/cli.py +++ b/cli.py @@ -5,7 +5,7 @@ from subprocess import call from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer -from path import Path +from pathlib import Path import time import os import yaml @@ -13,13 +13,13 @@ import logging.config -here = Path(__file__).abspath().dirname() +here = Path(__file__).absolute().parent logging.config.dictConfig(yaml.load((here / 'log.yaml').open(), Loader=yaml.FullLoader)) logger = logging.getLogger(__name__) -os.environ['PYTHONPATH'] = Path.getcwd() +os.environ['PYTHONPATH'] = str(Path.cwd()) def sh(cmd, all=False, **kwargs): @@ -35,7 +35,7 @@ def cli(): @cli.command() def antlr(): """generate a new parser based on the grammar using antlr""" - cwd = str(Path('qface/idl/parser').abspath()) + cwd = str(Path('qface/idl/parser').absolute()) sh('antlr4 -Dlanguage=Python3 -Werror -package qface.idl.parser -o . -listener -visitor T.g4', cwd=cwd) @@ -56,7 +56,7 @@ def test_ci(): @cli.command() def install(editable): """install the script onto the system using pip3""" - script_dir = str(Path(__file__).parent.abspath()) + script_dir = str(Path(__file__).parent.absolute()) click.secho(script_dir, fg='blue') if editable: sh('pip3 install --editable {0} --upgrade'.format(script_dir)) diff --git a/docs/extending.rst b/docs/extending.rst index d968c76..b83060d 100644 --- a/docs/extending.rst +++ b/docs/extending.rst @@ -158,7 +158,7 @@ The `RuleGenerator` allows you to extract the documentation rules into an extern .. code-block:: python from qface.generator import FileSystem, RuleGenerator - from path import Path + from pathlib import Path here = Path(__file__).dirname() diff --git a/qface/__about__.py b/qface/__about__.py index 3f83255..aa6883c 100644 --- a/qface/__about__.py +++ b/qface/__about__.py @@ -9,7 +9,7 @@ __title__ = "qface" __summary__ = "A generator framework based on a common modern IDL" __url__ = "https://pelagicore.github.io/qface/" -__version__ = "2.0.8" +__version__ = "2.0.9" __author__ = "JRyannel" __author_email__ = "" __copyright__ = "2019 Pelagicore" diff --git a/qface/app.py b/qface/app.py index 434195e..b4452d7 100644 --- a/qface/app.py +++ b/qface/app.py @@ -4,7 +4,7 @@ import sys import click import logging -from path import Path +from pathlib import Path from qface.generator import FileSystem, RuleGenerator from qface.watch import monitor from qface.utils import load_filters diff --git a/qface/cli.py b/qface/cli.py index f0b06b0..2f7334d 100644 --- a/qface/cli.py +++ b/qface/cli.py @@ -1,6 +1,6 @@ import sys import click -from path import Path +from pathlib import Path from qface.generator import FileSystem, RuleGenerator from qface.watch import monitor diff --git a/qface/contrib/logging.py b/qface/contrib/logging.py index 6b7625c..f9fb751 100644 --- a/qface/contrib/logging.py +++ b/qface/contrib/logging.py @@ -2,7 +2,7 @@ import logging import logging.config import coloredlogs -from path import Path +from pathlib import Path import os diff --git a/qface/filters.py b/qface/filters.py index 236c7b3..50f6c7b 100644 --- a/qface/filters.py +++ b/qface/filters.py @@ -20,7 +20,7 @@ def load_filters(path): return {} extra_filters = {} - spec = importlib.util.spec_from_file_location('filters', path.abspath()) + spec = importlib.util.spec_from_file_location('filters', path.absolute()) filters_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(filters_module) filters_module.get_filters(extra_filters) diff --git a/qface/generator.py b/qface/generator.py index d8e987f..9763774 100644 --- a/qface/generator.py +++ b/qface/generator.py @@ -4,7 +4,7 @@ from jinja2 import Environment, Template, Undefined, StrictUndefined from jinja2 import FileSystemLoader, PackageLoader, ChoiceLoader from jinja2 import TemplateSyntaxError, TemplateNotFound, TemplateError -from path import Path +from pathlib import Path from antlr4 import InputStream, FileStream, CommonTokenStream, ParseTreeWalker from antlr4.error import DiagnosticErrorListener, ErrorListener import shelve @@ -195,7 +195,7 @@ def _write(self, file_path: Path, template: str, context: dict, preserve: bool = force = self.force or force path = self.resolved_path / Path(self.apply(file_path, context)) if path.parent: - path.parent.makedirs_p() + path.parent.mkdir(parents=True, exist_ok=True) logger.info('write {0}'.format(path)) data = self.render(template, context) if self._has_different_content(data, path) or force: @@ -324,7 +324,7 @@ def _parse_document(document: Path, system: System = None, profile=EProfile.FULL logger.debug('parse document: {0}'.format(document)) stream = FileStream(str(document), encoding='utf-8') system = FileSystem._parse_stream(stream, system, document, profile) - FileSystem.merge_annotations(system, document.stripext() + '.yaml') + FileSystem.merge_annotations(system, os.path.splitext(document)[0] + '.yaml') return system @staticmethod @@ -351,10 +351,10 @@ def merge_annotations(system, document): return meta = FileSystem.load_yaml(document) if not meta: - click.secho('skipping empty: {0}'.format(document.name), fg='blue') + click.secho('skipping empty: {0}'.format(Path(document).name), fg='blue') return else: - click.secho('merge: {0}'.format(document.name), fg='blue') + click.secho('merge: {0}'.format(Path(document).name), fg='blue') try: for identifier, data in meta.items(): symbol = system.lookup(identifier) @@ -389,11 +389,11 @@ def parse(input, identifier: str = None, use_cache=False, clear_cache=True, patt system = cache[identifier] # if domain model not cached generate it for input in inputs: - path = Path.getcwd() / str(input) - if path.isfile(): + path = Path.cwd() / str(input) + if path.is_file(): FileSystem.parse_document(path, system) else: - for document in path.walkfiles(pattern): + for document in path.rglob(pattern): FileSystem.parse_document(document, system) if use_cache: diff --git a/qface/idl/domain.py b/qface/idl/domain.py index fbd0175..c8ffa5c 100644 --- a/qface/idl/domain.py +++ b/qface/idl/domain.py @@ -35,6 +35,7 @@ class System(object): """The root entity which consist of modules""" def __init__(self): log.debug('System()') + self._tags = dict() self._moduleMap = OrderedDict() # type: dict[str, Module] def __unicode__(self): @@ -43,6 +44,30 @@ def __unicode__(self): def __repr__(self): return '' + @property + def tags(self): + return self._tags + + def add_tag(self, tag): + """ add a tag to the tag list """ + if tag not in self._tags: + self._tags[tag] = dict() + + def add_attribute(self, tag, name, value): + """ add an attribute (nam, value pair) to the named tag """ + self.add_tag(tag) + d = self._tags[tag] + d[name] = value + + def tag(self, name): + """ return tag by name """ + return self._tags[name] + + def attribute(self, tag, name): + """ return attribute by tag and attribute name """ + if tag in self._tags and name in self._tags[tag]: + return self._tags[tag][name] + @property def modules(self): '''returns ordered list of module symbols''' @@ -53,6 +78,8 @@ def lookup(self, name: str): # if name in self._moduleMap: return self._moduleMap[name] + if name == 'system': + return self; # . (module_name, type_name, fragment_name) = self.split_typename(name) if not module_name in self._moduleMap: diff --git a/qface/idl/listener.py b/qface/idl/listener.py index db891f9..a529550 100644 --- a/qface/idl/listener.py +++ b/qface/idl/listener.py @@ -36,13 +36,14 @@ def escape_decode(s): class QFaceListener(TListener): def __init__(self, system, profile=EProfile.FULL): super().__init__() - click.secho('qface uses language profile: {}'.format(profile), fg='blue') + if profile != EProfile.FULL: + log.secho('qface uses language profile: {}'.format(profile), fg='blue') self.lang_features = get_features(profile) self.system = system or System() # type:System def check_support(self, feature, report=True): if feature not in self.lang_features and report: - click.secho('Unsuported language feature: {}'.format(EFeature.IMPORT), fg='red') + log.error('Unsuported language feature: {}'.format(EFeature.IMPORT), fg='red') return False return True diff --git a/qface/idl/parser/T4Lexer.py b/qface/idl/parser/T4Lexer.py index a3c7595..58b6ce2 100644 --- a/qface/idl/parser/T4Lexer.py +++ b/qface/idl/parser/T4Lexer.py @@ -210,7 +210,9 @@ class T4Lexer(Lexer): def __init__(self, input=None, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.10") + # Disable the version check for now as although there is a mismatch the Lexer seems to work fine. + # Rely on the weekly CI to make sure this keeps working also with later antlr versions. + # self.checkVersion("4.10") self._interp = LexerATNSimulator(self, self.atn, self.decisionsToDFA, PredictionContextCache()) self._actions = None self._predicates = None diff --git a/qface/idl/parser/T4Parser.py b/qface/idl/parser/T4Parser.py index df15673..4aa07b6 100644 --- a/qface/idl/parser/T4Parser.py +++ b/qface/idl/parser/T4Parser.py @@ -246,7 +246,9 @@ class T4Parser ( Parser ): def __init__(self, input:TokenStream, output:TextIO = sys.stdout): super().__init__(input, output) - self.checkVersion("4.10") + # Disable the version check for now as although there is a mismatch the Lexer seems to work fine. + # Rely on the weekly CI to make sure this keeps working also with later antlr versions. + # self.checkVersion("4.10") self._interp = ParserATNSimulator(self, self.atn, self.decisionsToDFA, self.sharedContextCache) self._predicates = None diff --git a/qface/watch.py b/qface/watch.py index 29bc30f..7a9762f 100644 --- a/qface/watch.py +++ b/qface/watch.py @@ -1,7 +1,7 @@ from watchdog.events import FileSystemEventHandler from watchdog.observers import Observer import click -from path import Path +from pathlib import Path import time from subprocess import call @@ -25,7 +25,7 @@ def run(self): if self.is_running: return self.is_running = True - call(self.args, cwd=Path.getcwd()) + call(self.args, cwd=Path.cwd()) self.is_running = False @@ -34,11 +34,11 @@ def monitor(args, watch): reloads the script given by argv when src files changes """ watch = watch if isinstance(watch, (list, tuple)) else [watch] - watch = [Path(entry).expand().abspath() for entry in watch] + watch = [Path(entry).expand().absolute() for entry in watch] event_handler = RunScriptChangeHandler(args) observer = Observer() for entry in watch: - if entry.isfile(): + if entry.is_file(): entry = entry.parent click.secho('watch recursive: {0}'.format(entry), fg='blue') observer.schedule(event_handler, entry, recursive=True) diff --git a/requirements.txt b/requirements.txt index 781a230..70019cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,10 @@ antlr4-python3-runtime>=4.7.1 argh>=0.26.2 click>=6.7 coloredlogs>=10.0 -humanfriendly>=4.15.1 Jinja2>=2.10.3 MarkupSafe>=1.0 -path.py>=11.0.1 -pathtools>=0.1.2 PyYAML>=5.1 six>=1.11.0 -watchdog>=0.8.3 +watchdog>=1.0 pytest>=5.3.5 pytest-cov>=2.8.1 diff --git a/requirements_minimal.txt b/requirements_minimal.txt index f06ff91..14d1a56 100644 --- a/requirements_minimal.txt +++ b/requirements_minimal.txt @@ -2,13 +2,10 @@ antlr4-python3-runtime==4.7.1 argh==0.26.2 click==6.7 coloredlogs==10.0 -humanfriendly==4.15.1 Jinja2==2.10.3 MarkupSafe==1.1 -path.py==11.0.1 -pathtools==0.1.2 PyYAML==5.1 six==1.11.0 -watchdog==0.8.3 +watchdog==1.0 pytest==6.2.5 pytest-cov==2.8.1 diff --git a/setup.py b/setup.py index bb9fbbd..3dca1d3 100644 --- a/setup.py +++ b/setup.py @@ -49,9 +49,8 @@ 'click>=6.7', 'antlr4-python3-runtime>=4.7.1', 'jinja2>=2.10.3', - 'path.py>=11.0.1', 'pyyaml>=5.1', - 'watchdog>=0.8.3', + 'watchdog>=1.0', 'six>=1.11.0', 'coloredlogs>=10.0', ], diff --git a/tests/in/com.pelagicore.ivi.tuner.yaml b/tests/in/com.pelagicore.ivi.tuner.yaml index a320743..6eae90b 100644 --- a/tests/in/com.pelagicore.ivi.tuner.yaml +++ b/tests/in/com.pelagicore.ivi.tuner.yaml @@ -1,3 +1,6 @@ +system: + global_option: true + com.pelagicore.ivi.tuner.Tuner: port: 12345 config: diff --git a/tests/test_climate.py b/tests/test_climate.py index 1a2135a..b7adec6 100644 --- a/tests/test_climate.py +++ b/tests/test_climate.py @@ -1,7 +1,7 @@ from qface.generator import FileSystem import logging import logging.config -from path import Path +from pathlib import Path # logging.config.fileConfig('logging.ini') logging.basicConfig() @@ -9,7 +9,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def load_system(): diff --git a/tests/test_comments.py b/tests/test_comments.py index d2f18b6..6af3d1c 100644 --- a/tests/test_comments.py +++ b/tests/test_comments.py @@ -2,7 +2,7 @@ from qface.helper import doc import logging import logging.config -from path import Path +from pathlib import Path # logging.config.fileConfig('logging.ini') diff --git a/tests/test_generator.py b/tests/test_generator.py index f83fb1b..9e1d53c 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -4,7 +4,8 @@ import logging import logging.config import tempfile -from path import Path +from pathlib import Path +import shutil # logging.config.fileConfig('logging.ini') logging.basicConfig() @@ -12,7 +13,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def loadSystem(): @@ -62,16 +63,16 @@ def test_parse_document_mixed(): def test_destination_prefix(): system = FileSystem.parse(inputPath) out = Path('tests/out') - out.rmtree_p() - out.makedirs_p() + shutil.rmtree(out, ignore_errors=True) + out.mkdir(parents=True, exist_ok=True) generator = Generator(search_path='tests/templates') for module in system.modules: dst_template = '{{out}}/{{module|lower}}.txt' - ctx = {'out': out.abspath(), 'module': module} + ctx = {'out': out.absolute(), 'module': module} generator.write(dst_template, 'module.txt', ctx) path = generator.apply(dst_template, ctx) assert Path(path).exists() == True - out.rmtree_p() + shutil.rmtree(out, ignore_errors=True) @patch('sys.stderr', new_callable=StringIO) def test_error_template_syntax_error(mock_stderr): diff --git a/tests/test_json.py b/tests/test_json.py index 785680a..827e140 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -1,6 +1,6 @@ from qface.generator import FileSystem import logging -from path import Path +from pathlib import Path import json # logging.config.fileConfig('logging.ini') diff --git a/tests/test_lookup.py b/tests/test_lookup.py index 47184c0..5d27536 100644 --- a/tests/test_lookup.py +++ b/tests/test_lookup.py @@ -1,6 +1,6 @@ import logging import logging.config -from path import Path +from pathlib import Path from qface.generator import FileSystem @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def load_tuner(): diff --git a/tests/test_parser.py b/tests/test_parser.py index 324b68a..ca9277d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,7 +1,7 @@ import logging import logging.config import pytest -from path import Path +from pathlib import Path from qface.generator import FileSystem import qface.idl.domain as domain @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def load_tuner(): diff --git a/tests/test_qtcpp_helper.py b/tests/test_qtcpp_helper.py index 3a12b0f..39cd190 100644 --- a/tests/test_qtcpp_helper.py +++ b/tests/test_qtcpp_helper.py @@ -1,6 +1,6 @@ import logging import logging.config -from path import Path +from pathlib import Path from qface.helper import qtcpp from qface.generator import FileSystem diff --git a/tests/test_tags.py b/tests/test_tags.py index be9483c..8cf340b 100644 --- a/tests/test_tags.py +++ b/tests/test_tags.py @@ -3,7 +3,7 @@ from io import StringIO import logging import logging.config -from path import Path +from pathlib import Path # logging.config.fileConfig('logging.ini') @@ -12,7 +12,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def loadTuner(): @@ -41,6 +41,9 @@ def test_tag(): assert 'default' in enum.tags assert enum.attribute('default', 'value') == 'FM' + # lookup system + system = system.lookup('system') + assert 'global_option' in system.tags def test_meta_tags(): system = loadTuner() diff --git a/tests/test_validation.py b/tests/test_validation.py index d742872..a420783 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,6 +1,6 @@ import logging import logging.config -from path import Path +from pathlib import Path from qface.generator import FileSystem @@ -10,7 +10,7 @@ log = logging.getLogger(__name__) inputPath = Path('tests/in') -log.debug('input path folder: {0}'.format(inputPath.abspath())) +log.debug('input path folder: {0}'.format(inputPath.absolute())) def load_one(): diff --git a/tests/test_values.py b/tests/test_values.py index 9ce93d1..e2d2071 100644 --- a/tests/test_values.py +++ b/tests/test_values.py @@ -1,7 +1,7 @@ from qface.generator import FileSystem import logging import logging.config -from path import Path +from pathlib import Path # logging.config.fileConfig('logging.ini')