diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 47abe1ec..b8dda20e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,7 @@ Change log ========== 0.19.10 (2021-11-05) -------------------- +-------------------- New features ~~~~~~~~~~~~ diff --git a/doc/args.rst b/doc/args.rst index 8443e35a..c8a99c0e 100644 --- a/doc/args.rst +++ b/doc/args.rst @@ -34,5 +34,9 @@ add comments on found issues right to the selected code review system. | * -f='test 1:!unit test 1' - run all steps with 'test 1' substring in their names except those containing 'unit test 1' + --html-log -hl : @after + To make sure all the interactive features of such a page work right in Jenkins artifacts, + please refer to the following :doc:`guide ` + {init,run,poll,submit,github-handler} : @replace | See detailed description of additional commands :doc:`here `. diff --git a/doc/index.rst b/doc/index.rst index de7e8b40..a2db8596 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,6 +14,7 @@ Project 'Universum' github_handler.rst universum_docs.rst teamcity.rst + jenkins.rst examples.rst changelog_ref.rst diff --git a/doc/jenkins.rst b/doc/jenkins.rst new file mode 100644 index 00000000..f22caa5f --- /dev/null +++ b/doc/jenkins.rst @@ -0,0 +1,137 @@ +Integration with Jenkins +======================== + +`Universum` requires no special integration with `Jenkins`, except for one thing. It is usually launched +as one long step in a single build stage. Because of that, the whole `Universum` log is printed as a plain text +without navigation anchors. + +Compare with these: + +* When ran locally, all Universum build step logs are stored to separate files instead of printing their + output to console +* When launched by TeamCity, `Unviersum` uses `service messages + `__ to increase log readability + +To simplify navigating long logs and finding relevant information on build results, we provide a user-friendly +interactive log with collapsible blocks and other features. This log can also be used outside of Jenkins, but +for Jenkins users it adds the missing functionality. + +.. warning:: + + By default, Jenkins `does not render interactive content `__. + This means that without changing server settings, interactive features of the generated log will be + inaccessible when opened directly from Jenkins artifacts. + + +Here's the list of steps we performed to integrate the interactive log with Jenkins server: + +1. :ref:`Add a command line option ` to generate a self-contained HTML file +2. :ref:`Add a Jenkins plugin ` to integrate a generated log into a Jenkins job +3. :ref:`Set up Resource Root URL ` to allow Jenkins rendering interactive content +4. :ref:`Configure reverse proxy ` to handle multiple domains interaction + + +.. _jenkins#add_arg: + +Add command line option +----------------------- + +To generate a single interactive self-contained HTML file, pass an `--html-log `__ +option to command line. It will be stored to in project `artifacts `__ folder. +Note that the log name can either be specified or left default. + +.. note:: + + Jenkins jobs do not show any files via web interface if not not specifically configured to do so. Use + ``archiveArtifacts`` in Pipeline or ``Archive the artifacts`` in `Post-build Actions` to check the file presence + and contents before next step if needed + + +.. _jenkins#plugin: + +Add a Jenkins plugin +-------------------- + +A `HTML Publisher `__ plugin is a very convenient way to add an HTML report +to a Jenkins job. To use it, you will need to: + +1. Install it on server (server restart might be needed to apply changes) +2. Pass the generated log name to plugin configuration as described in manual (https://plugins.jenkins.io/htmlpublisher/) + via `Post-build Actions` or Pipeline +3. Launch a configured job at least once for the log to be generated +4. Let Jenkins render the interactive content of log (see the next section) + + +.. _jenkins#resource_url: + +Set up Resource Root URL +------------------------ + +As already mentioned above, due to `Jenkins Content-Security-Policy +`__ some features of an interactive log +might not work properly, and its contents might be displayed incorrectly. + +A recommended way to allow Jenkins server to render interactive user content is to `configure Jenkins to publish +artifacts on a different domain `__ +by changing ``Resource Root URL`` in `Manage Jenkins » Configure System » Serve resource files from another domain` +from ``
`` to ```` (e.g. ``my.jenkins.io`` to ``res.my.jenkins.io``). + +Note that Jenkins interaction with resource domain, resolved to the same host IP is not done via ``localhost`` +network interface. The reason for that is Jenkins requiring some interaction with itself via this domain name. +This means that both ``
`` and ```` domain names must be resolved correctly, either +globally (via DNS) or locally on both client and server machines (via ``/etc/hosts`` files). The correctness of +name resolving is checked when saving the changes to this setting; but Jenkins will only show warning, and not +fail if domain name is not resolved. + +.. note:: + + If main server domain name is not resolved using DNS, ``/etc/hosts`` or any other means, the web-interface + will only be accessible via IP, and not the name. Because of that, the Jenkins interaction with itself + via domain name will fail because host name is not passed to the Jenkins server + +Here are the symptoms of domain names not resolving correctly: + +1. Jenkins warnings when trying to save the updated settings +2. Client inability to access said pages (timeout error) + +To set up domain name resolving, add following lines to server ``/ets/host`` file:: + + 127.0.0.1
+ + +And add the following lines to client ``/ets/host`` file:: + +
+ + + +.. _jenkins#nginx: + +Configure reverse proxy +----------------------- + +Note that this step is only needed if `Nginx reverse proxy +`__ is used. + +To understand why these fixes are needed, let's pay more attention to the mechanism of 'another domain', used by +Jenkins. When requesting an artifact, that is served from another domain, user first goes to main Jenkins web +server, that returns a redirection link to acquire a said artifact. + +As `specified in docs `__, +without specification Nginx replaces ``Host`` header with ``$proxy_host``. In this case it changes +```` to proxy IP specifications. The problem is that without the Host header the Jenkins server +is not able to understand that the request is sent to the resource domain. Therefore it returns the 404 NOT FOUND error. + +To pass them correctly, adjust the configuration as instructed in manual mentioned above. Add the following lines +to Nginx configuration file:: + + location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + +so that real headers are passed to Jenkins to handle the resource domain magic. + +.. note:: + + Also you might need to set ``server_name`` to ``
`` (whitespace separated) diff --git a/tests/test_html_output.py b/tests/test_html_output.py index 8388f846..3d740446 100644 --- a/tests/test_html_output.py +++ b/tests/test_html_output.py @@ -61,9 +61,9 @@ def browser(): def test_cli_log_custom_name(tmpdir): - custom_log_name = "custom_name" + custom_log_name = "custom_name.html" artifact_dir = check_cli(tmpdir, ["-hl", custom_log_name]) - assert os.path.exists(os.path.join(artifact_dir, f"{custom_log_name}.html")) + assert os.path.exists(os.path.join(artifact_dir, custom_log_name)) def test_cli_log_default_name(tmpdir): @@ -90,7 +90,7 @@ def test_success_clean_build(tmpdir, browser): def check_html_log(artifact_dir, browser): - log_path = os.path.join(artifact_dir, f"{log_name}.html") + log_path = os.path.join(artifact_dir, log_name) assert os.path.exists(log_path) browser.get(f"file://{log_path}") diff --git a/universum/modules/output/html_output.py b/universum/modules/output/html_output.py index 6f440302..b25d69e2 100644 --- a/universum/modules/output/html_output.py +++ b/universum/modules/output/html_output.py @@ -11,7 +11,7 @@ class HtmlOutput(BaseOutput): - default_name = "universum_log" + default_name = "universum_log.html" def __init__(self, *args, log_name=default_name, **kwargs): super().__init__(*args, **kwargs) @@ -23,7 +23,7 @@ def __init__(self, *args, log_name=default_name, **kwargs): self.module_dir = os.path.dirname(os.path.realpath(__file__)) def set_artifact_dir(self, artifact_dir): - self._log_path = os.path.join(artifact_dir, f"{self._log_name}.html") + self._log_path = os.path.join(artifact_dir, self._log_name) def open_block(self, num_str, name): opening_html = f'' + \ diff --git a/universum/modules/output/output.py b/universum/modules/output/output.py index 438c13a3..c6979e95 100644 --- a/universum/modules/output/output.py +++ b/universum/modules/output/output.py @@ -22,23 +22,20 @@ class Output(Module): terminal_driver_factory = Dependency(TerminalBasedOutput) html_driver_factory = Dependency(HtmlOutput) - html_log_disabled_arg_value = None - @staticmethod def define_arguments(argument_parser): parser = argument_parser.get_or_create_group("Output", "Log appearance parameters") parser.add_argument("--out-type", "-ot", dest="type", choices=["tc", "term", "jenkins"], help="Type of output to produce (tc - TeamCity, jenkins - Jenkins, term - terminal). " - "TeamCity and Jenkins environments are detected automatically when launched on build " - "agent.") + "TeamCity and Jenkins environments are detected automatically " + "when launched on build agent.") # `universum` -> html_log == default # `universum -hl` -> html_log == const # `universum -hl custom` -> html_log == custom - parser.add_argument("--html-log", "-hl", - nargs="?", const=HtmlOutput.default_name, default=Output.html_log_disabled_arg_value, - help="Generate self-contained HTML log in artifacts directory. " - "You may specify a file name in this parameter's value or default " - "one will be used") + parser.add_argument("--html-log", "-hl", nargs="?", const=HtmlOutput.default_name, default=None, + help=f"Generate a self-contained user-friendly HTML log. " + f"Pass a desired log name as a parameter to this option, or a default " + f"'{HtmlOutput.default_name}' will be used.") def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -103,13 +100,12 @@ def log_execution_finish(self, title: str, version: str) -> None: self.html_driver.log_execution_finish(title, version) def _create_html_driver(self): - is_enabled = self.settings.html_log != self.html_log_disabled_arg_value + is_enabled = self.settings.html_log is not None html_driver = self.html_driver_factory(log_name=self.settings.html_log) if is_enabled else None handler = HtmlDriverHandler(html_driver) return handler - class HasOutput(Module): out_factory: ClassVar = Dependency(Output)