diff --git a/Makefile b/Makefile index 41c270bb..bad63742 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,5 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + cp _build/html/_images/sphx_glr_example_job_submit_api_thumb.png _build/html/_images/sphx_glr_example_job_info_api_thumb.png \ No newline at end of file diff --git a/auto_examples/auto_examples_jupyter.zip b/auto_examples/auto_examples_jupyter.zip index 13b1be2a..1b2b11e7 100644 Binary files a/auto_examples/auto_examples_jupyter.zip and b/auto_examples/auto_examples_jupyter.zip differ diff --git a/auto_examples/auto_examples_python.zip b/auto_examples/auto_examples_python.zip index 6ef22f8a..8b168963 100644 Binary files a/auto_examples/auto_examples_python.zip and b/auto_examples/auto_examples_python.zip differ diff --git a/auto_examples/example_job_info_api.ipynb b/auto_examples/example_job_info_api.ipynb new file mode 100644 index 00000000..3df22dbc --- /dev/null +++ b/auto_examples/example_job_info_api.ipynb @@ -0,0 +1,259 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# Introductory example - Job Info API\n\nThis example will show how to get information\nabout a job after the fact.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import json\nimport time\nimport os\nimport flux\nfrom flux.job import JobspecV1\nimport subprocess" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we instantiate a flux handle. This will connect to the running flux instance.\nIf you were running this on a cluster with Flux, you'd likely already be able to\nconnect. If you are testing out on your own, you might need to do flux start --test-size=4\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "handle = flux.Flux()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is a new jobspec, or a recipe for a flux job. You'll notice we are providing a command\ndirectly, along with tasks, nodes, and cores per task. You could also provide a script here.\nIf we were doing this on the command line, it would be equivalent to what is generated by:\nflux submit --ntasks=4 --nodes=2 --cores-per-task=2 sleep 10\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobspec = JobspecV1.from_command(\n command=[\"hostname\"], num_tasks=1, num_nodes=1, cores_per_task=1\n)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is how we set the \"current working directory\" (cwd) for the job\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobspec.cwd = os.getcwd()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is how we set the job environment\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobspec.environment = dict(os.environ)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's submit the job! We will get the job id.\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "jobid = flux.job.submit(handle, jobspec)\ntime.sleep(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's say we store that jobid somewhere how do we get info later?\nWe know that if we ran flux jobs -a on the command line, we'd see the job\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "res = subprocess.getoutput('flux jobs -a')\nprint(res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And if you are an expert user, you know that you can see metadata for a job\nThis command, without a key, will show the keys available to you\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "res = subprocess.getoutput(f'flux job info {jobid} | true')\nprint(res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And since the underlying logic here is pinging the flux KVS or key value store,\nwe can select one of those keys to view. For example, here is the jobspec\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "res = subprocess.getoutput(f'flux job info {jobid} jobspec')\nprint(res)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is great, but ideally we can get this metadata directly from Python.\nFirst, here is a way to get basic jobinfo. Given we start with a string jobid,\nwe will first want to parse it back into a Flux JobID, and then prepare\na payload to the Job List RPC to say \"give me all the attributes back\"\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "fluxjob = flux.job.JobID(jobid)\npayload = {\"id\": fluxjob, \"attrs\": [\"all\"]}\nrpc = flux.job.list.JobListIdRPC(handle, \"job-list.list-id\", payload)\njobinfo = rpc.get_job()\nprint(json.dumps(jobinfo, indent=4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can get less commonly used (and thus exposed) metadata like this\nsuch as the emoji state!\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "info = rpc.get_jobinfo()\nprint(info.__dict__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But for either of the above approaches, we aren't getting anything back about our\noriginal jobspec! That's because we need to query the KVS for that. Notice here we\nhave metadata like the current working directory (cwd)\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "kvs = flux.job.job_kvs(handle, jobid)\njobspec = kvs.get('jobspec')\nprint(json.dumps(jobspec))\ntime.sleep(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, to watch (or stream) output, you can do the following.\nEach line here is a json structure that you can further parse.\nAs an example, if \"data\" is present as a key, this usually is output\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "for line in flux.job.event_watch(handle, jobid, \"guest.output\"):\n print(line)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/auto_examples/example_job_info_api.py b/auto_examples/example_job_info_api.py new file mode 100644 index 00000000..4ba3d335 --- /dev/null +++ b/auto_examples/example_job_info_api.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +Introductory example - Job Info API +=================================== + +This example will show how to get information +about a job after the fact. +""" + + +import json +import time +import os +import flux +from flux.job import JobspecV1 +import subprocess + +#%% +# Here we instantiate a flux handle. This will connect to the running flux instance. +# If you were running this on a cluster with Flux, you'd likely already be able to +# connect. If you are testing out on your own, you might need to do flux start --test-size=4 +handle = flux.Flux() + +#%% +# This is a new jobspec, or a recipe for a flux job. You'll notice we are providing a command +# directly, along with tasks, nodes, and cores per task. You could also provide a script here. +# If we were doing this on the command line, it would be equivalent to what is generated by: +# flux submit --ntasks=4 --nodes=2 --cores-per-task=2 sleep 10 +jobspec = JobspecV1.from_command( + command=["hostname"], num_tasks=1, num_nodes=1, cores_per_task=1 +) + +#%% +# This is how we set the "current working directory" (cwd) for the job +jobspec.cwd = os.getcwd() + +#%% +# This is how we set the job environment +jobspec.environment = dict(os.environ) + +#%% +# Let's submit the job! We will get the job id. +jobid = flux.job.submit(handle, jobspec) +time.sleep(2) + +#%% +# Now let's say we store that jobid somewhere how do we get info later? +# We know that if we ran flux jobs -a on the command line, we'd see the job +res = subprocess.getoutput('flux jobs -a') +print(res) + +#%% +# And if you are an expert user, you know that you can see metadata for a job +# This command, without a key, will show the keys available to you +res = subprocess.getoutput(f'flux job info {jobid} | true') +print(res) + +#%% +# And since the underlying logic here is pinging the flux KVS or key value store, +# we can select one of those keys to view. For example, here is the jobspec +res = subprocess.getoutput(f'flux job info {jobid} jobspec') +print(res) + +#%% +# This is great, but ideally we can get this metadata directly from Python. +# First, here is a way to get basic jobinfo. Given we start with a string jobid, +# we will first want to parse it back into a Flux JobID, and then prepare +# a payload to the Job List RPC to say "give me all the attributes back" +fluxjob = flux.job.JobID(jobid) +payload = {"id": fluxjob, "attrs": ["all"]} +rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) +jobinfo = rpc.get_job() +print(json.dumps(jobinfo, indent=4)) + +#%% +# You can get less commonly used (and thus exposed) metadata like this +# such as the emoji state! +info = rpc.get_jobinfo() +print(info.__dict__) + +#%% +# But for either of the above approaches, we aren't getting anything back about our +# original jobspec! That's because we need to query the KVS for that. Notice here we +# have metadata like the current working directory (cwd) +kvs = flux.job.job_kvs(handle, jobid) +jobspec = kvs.get('jobspec') +print(json.dumps(jobspec)) +time.sleep(2) + +#%% +# Finally, to watch (or stream) output, you can do the following. +# Each line here is a json structure that you can further parse. +# As an example, if "data" is present as a key, this usually is output +for line in flux.job.event_watch(handle, jobid, "guest.output"): + print(line) + + + diff --git a/auto_examples/example_job_info_api.py.md5 b/auto_examples/example_job_info_api.py.md5 new file mode 100644 index 00000000..c06d4931 --- /dev/null +++ b/auto_examples/example_job_info_api.py.md5 @@ -0,0 +1 @@ +553b1d62e9c0adeedf1804adf3134c46 \ No newline at end of file diff --git a/auto_examples/example_job_info_api.rst b/auto_examples/example_job_info_api.rst new file mode 100644 index 00000000..5a8f0448 --- /dev/null +++ b/auto_examples/example_job_info_api.rst @@ -0,0 +1,388 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "auto_examples/example_job_info_api.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + :ref:`Go to the end ` + to download the full example code + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_auto_examples_example_job_info_api.py: + + +Introductory example - Job Info API +=================================== + +This example will show how to get information +about a job after the fact. + +.. GENERATED FROM PYTHON SOURCE LINES 9-18 + +.. code-block:: default + + + + import json + import time + import os + import flux + from flux.job import JobspecV1 + import subprocess + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 19-22 + +Here we instantiate a flux handle. This will connect to the running flux instance. +If you were running this on a cluster with Flux, you'd likely already be able to +connect. If you are testing out on your own, you might need to do flux start --test-size=4 + +.. GENERATED FROM PYTHON SOURCE LINES 22-24 + +.. code-block:: default + + handle = flux.Flux() + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 25-29 + +This is a new jobspec, or a recipe for a flux job. You'll notice we are providing a command +directly, along with tasks, nodes, and cores per task. You could also provide a script here. +If we were doing this on the command line, it would be equivalent to what is generated by: +flux submit --ntasks=4 --nodes=2 --cores-per-task=2 sleep 10 + +.. GENERATED FROM PYTHON SOURCE LINES 29-33 + +.. code-block:: default + + jobspec = JobspecV1.from_command( + command=["hostname"], num_tasks=1, num_nodes=1, cores_per_task=1 + ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 34-35 + +This is how we set the "current working directory" (cwd) for the job + +.. GENERATED FROM PYTHON SOURCE LINES 35-37 + +.. code-block:: default + + jobspec.cwd = os.getcwd() + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 38-39 + +This is how we set the job environment + +.. GENERATED FROM PYTHON SOURCE LINES 39-41 + +.. code-block:: default + + jobspec.environment = dict(os.environ) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 42-43 + +Let's submit the job! We will get the job id. + +.. GENERATED FROM PYTHON SOURCE LINES 43-46 + +.. code-block:: default + + jobid = flux.job.submit(handle, jobspec) + time.sleep(2) + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 47-49 + +Now let's say we store that jobid somewhere how do we get info later? +We know that if we ran flux jobs -a on the command line, we'd see the job + +.. GENERATED FROM PYTHON SOURCE LINES 49-52 + +.. code-block:: default + + res = subprocess.getoutput('flux jobs -a') + print(res) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + JOBID USER NAME ST NTASKS NNODES TIME INFO + ƒkFU3pT vscode hostname CD 1 1 0.028s 610b9ad16799 + + + + +.. GENERATED FROM PYTHON SOURCE LINES 53-55 + +And if you are an expert user, you know that you can see metadata for a job +This command, without a key, will show the keys available to you + +.. GENERATED FROM PYTHON SOURCE LINES 55-58 + +.. code-block:: default + + res = subprocess.getoutput(f'flux job info {jobid} | true') + print(res) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Missing lookup key(s), common keys: + J + R + eventlog + jobspec + guest.exec.eventlog + guest.input + guest.output + + + + +.. GENERATED FROM PYTHON SOURCE LINES 59-61 + +And since the underlying logic here is pinging the flux KVS or key value store, +we can select one of those keys to view. For example, here is the jobspec + +.. GENERATED FROM PYTHON SOURCE LINES 61-64 + +.. code-block:: default + + res = subprocess.getoutput(f'flux job info {jobid} jobspec') + print(res) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + {"resources":[{"type":"node","count":1,"with":[{"type":"slot","count":1,"with":[{"type":"core","count":1}],"label":"task"}]}],"tasks":[{"command":["hostname"],"slot":"task","count":{"per_slot":1}}],"attributes":{"system":{"duration":0,"cwd":"/workspaces/flux-docs/examples"}},"version":1} + + + + +.. GENERATED FROM PYTHON SOURCE LINES 65-69 + +This is great, but ideally we can get this metadata directly from Python. +First, here is a way to get basic jobinfo. Given we start with a string jobid, +we will first want to parse it back into a Flux JobID, and then prepare +a payload to the Job List RPC to say "give me all the attributes back" + +.. GENERATED FROM PYTHON SOURCE LINES 69-75 + +.. code-block:: default + + fluxjob = flux.job.JobID(jobid) + payload = {"id": fluxjob, "attrs": ["all"]} + rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) + jobinfo = rpc.get_job() + print(json.dumps(jobinfo, indent=4)) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + { + "id": 28387049472, + "userid": 1000, + "urgency": 16, + "priority": 16, + "t_submit": 1682482856.185918, + "t_depend": 1682482856.1989403, + "t_run": 1682482856.2132757, + "t_cleanup": 1682482856.2413428, + "t_inactive": 1682482856.2439826, + "state": 64, + "name": "hostname", + "ntasks": 1, + "ncores": 1, + "duration": 0.0, + "nnodes": 1, + "ranks": "0", + "nodelist": "610b9ad16799", + "success": true, + "exception_occurred": false, + "result": 1, + "expiration": 4836082856.0, + "waitstatus": 0 + } + + + + +.. GENERATED FROM PYTHON SOURCE LINES 76-78 + +You can get less commonly used (and thus exposed) metadata like this +such as the emoji state! + +.. GENERATED FROM PYTHON SOURCE LINES 78-81 + +.. code-block:: default + + info = rpc.get_jobinfo() + print(info.__dict__) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + {'_t_depend': 1682482856.1989403, '_t_run': 1682482856.2132757, '_t_cleanup': 1682482856.2413428, '_t_inactive': 1682482856.2439826, '_duration': 0.0, '_expiration': 4836082856.0, '_name': 'hostname', '_queue': '', '_ntasks': 1, '_ncores': 1, '_nnodes': 1, '_priority': 16, '_ranks': '0', '_nodelist': '610b9ad16799', '_success': True, '_waitstatus': 0, '_id': JobID(28387049472), '_userid': 1000, '_urgency': 16, '_t_submit': 1682482856.185918, '_exception_occurred': False, '_state_id': 64, '_result_id': 1, '_exception': , '_annotations': , '_sched': , '_user': , '_dependencies': []} + + + + +.. GENERATED FROM PYTHON SOURCE LINES 82-85 + +But for either of the above approaches, we aren't getting anything back about our +original jobspec! That's because we need to query the KVS for that. Notice here we +have metadata like the current working directory (cwd) + +.. GENERATED FROM PYTHON SOURCE LINES 85-90 + +.. code-block:: default + + kvs = flux.job.job_kvs(handle, jobid) + jobspec = kvs.get('jobspec') + print(json.dumps(jobspec)) + time.sleep(2) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + {"resources": [{"type": "node", "count": 1, "with": [{"type": "slot", "count": 1, "with": [{"type": "core", "count": 1}], "label": "task"}]}], "tasks": [{"command": ["hostname"], "slot": "task", "count": {"per_slot": 1}}], "attributes": {"system": {"duration": 0, "cwd": "/workspaces/flux-docs/examples"}}, "version": 1} + + + + +.. GENERATED FROM PYTHON SOURCE LINES 91-94 + +Finally, to watch (or stream) output, you can do the following. +Each line here is a json structure that you can further parse. +As an example, if "data" is present as a key, this usually is output + +.. GENERATED FROM PYTHON SOURCE LINES 94-99 + +.. code-block:: default + + for line in flux.job.event_watch(handle, jobid, "guest.output"): + print(line) + + + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + 1682482856.23028: header {'version': 1, 'encoding': {'stdout': 'UTF-8', 'stderr': 'UTF-8'}, 'count': {'stdout': 1, 'stderr': 1}, 'options': {}} + 1682482856.23757: data {'stream': 'stderr', 'rank': '0', 'eof': True} + 1682482856.23760: data {'stream': 'stdout', 'rank': '0', 'data': '610b9ad16799\n'} + 1682482856.23763: data {'stream': 'stdout', 'rank': '0', 'eof': True} + + + + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** ( 0 minutes 4.381 seconds) + + +.. _sphx_glr_download_auto_examples_example_job_info_api.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + + + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: example_job_info_api.py ` + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: example_job_info_api.ipynb ` + + +.. only:: html + + .. rst-class:: sphx-glr-signature + + `Gallery generated by Sphinx-Gallery `_ diff --git a/auto_examples/example_job_info_api_codeobj.pickle b/auto_examples/example_job_info_api_codeobj.pickle new file mode 100644 index 00000000..1d5d5c4f Binary files /dev/null and b/auto_examples/example_job_info_api_codeobj.pickle differ diff --git a/auto_examples/images/thumb/sphx_glr_example_job_info_api_thumb.png b/auto_examples/images/thumb/sphx_glr_example_job_info_api_thumb.png new file mode 100644 index 00000000..d1854d39 Binary files /dev/null and b/auto_examples/images/thumb/sphx_glr_example_job_info_api_thumb.png differ diff --git a/auto_examples/index.rst b/auto_examples/index.rst index be13a054..5f344c70 100644 --- a/auto_examples/index.rst +++ b/auto_examples/index.rst @@ -40,6 +40,23 @@ of using the Flux Python API. +.. raw:: html + +
+ +.. only:: html + + .. image:: /auto_examples/images/thumb/sphx_glr_example_job_info_api_thumb.png + :alt: + + :ref:`sphx_glr_auto_examples_example_job_info_api.py` + +.. raw:: html + +
Introductory example - Job Info API
+
+ + .. raw:: html @@ -49,6 +66,7 @@ of using the Flux Python API. :hidden: /auto_examples/example_job_submit_api + /auto_examples/example_job_info_api .. only:: html diff --git a/auto_examples/sg_execution_times.rst b/auto_examples/sg_execution_times.rst index edc588df..97e10eb0 100644 --- a/auto_examples/sg_execution_times.rst +++ b/auto_examples/sg_execution_times.rst @@ -3,10 +3,13 @@ .. _sphx_glr_auto_examples_sg_execution_times: + Computation times ================= -**00:00.232** total execution time for **auto_examples** files: +**00:04.381** total execution time for **auto_examples** files: +-----------------------------------------------------------------------------------------+-----------+--------+ -| :ref:`sphx_glr_auto_examples_example_job_submit_api.py` (``example_job_submit_api.py``) | 00:00.232 | 0.0 MB | +| :ref:`sphx_glr_auto_examples_example_job_info_api.py` (``example_job_info_api.py``) | 00:04.381 | 0.0 MB | ++-----------------------------------------------------------------------------------------+-----------+--------+ +| :ref:`sphx_glr_auto_examples_example_job_submit_api.py` (``example_job_submit_api.py``) | 00:00.000 | 0.0 MB | +-----------------------------------------------------------------------------------------+-----------+--------+ diff --git a/examples/example_job_info_api.py b/examples/example_job_info_api.py new file mode 100644 index 00000000..4ba3d335 --- /dev/null +++ b/examples/example_job_info_api.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +Introductory example - Job Info API +=================================== + +This example will show how to get information +about a job after the fact. +""" + + +import json +import time +import os +import flux +from flux.job import JobspecV1 +import subprocess + +#%% +# Here we instantiate a flux handle. This will connect to the running flux instance. +# If you were running this on a cluster with Flux, you'd likely already be able to +# connect. If you are testing out on your own, you might need to do flux start --test-size=4 +handle = flux.Flux() + +#%% +# This is a new jobspec, or a recipe for a flux job. You'll notice we are providing a command +# directly, along with tasks, nodes, and cores per task. You could also provide a script here. +# If we were doing this on the command line, it would be equivalent to what is generated by: +# flux submit --ntasks=4 --nodes=2 --cores-per-task=2 sleep 10 +jobspec = JobspecV1.from_command( + command=["hostname"], num_tasks=1, num_nodes=1, cores_per_task=1 +) + +#%% +# This is how we set the "current working directory" (cwd) for the job +jobspec.cwd = os.getcwd() + +#%% +# This is how we set the job environment +jobspec.environment = dict(os.environ) + +#%% +# Let's submit the job! We will get the job id. +jobid = flux.job.submit(handle, jobspec) +time.sleep(2) + +#%% +# Now let's say we store that jobid somewhere how do we get info later? +# We know that if we ran flux jobs -a on the command line, we'd see the job +res = subprocess.getoutput('flux jobs -a') +print(res) + +#%% +# And if you are an expert user, you know that you can see metadata for a job +# This command, without a key, will show the keys available to you +res = subprocess.getoutput(f'flux job info {jobid} | true') +print(res) + +#%% +# And since the underlying logic here is pinging the flux KVS or key value store, +# we can select one of those keys to view. For example, here is the jobspec +res = subprocess.getoutput(f'flux job info {jobid} jobspec') +print(res) + +#%% +# This is great, but ideally we can get this metadata directly from Python. +# First, here is a way to get basic jobinfo. Given we start with a string jobid, +# we will first want to parse it back into a Flux JobID, and then prepare +# a payload to the Job List RPC to say "give me all the attributes back" +fluxjob = flux.job.JobID(jobid) +payload = {"id": fluxjob, "attrs": ["all"]} +rpc = flux.job.list.JobListIdRPC(handle, "job-list.list-id", payload) +jobinfo = rpc.get_job() +print(json.dumps(jobinfo, indent=4)) + +#%% +# You can get less commonly used (and thus exposed) metadata like this +# such as the emoji state! +info = rpc.get_jobinfo() +print(info.__dict__) + +#%% +# But for either of the above approaches, we aren't getting anything back about our +# original jobspec! That's because we need to query the KVS for that. Notice here we +# have metadata like the current working directory (cwd) +kvs = flux.job.job_kvs(handle, jobid) +jobspec = kvs.get('jobspec') +print(json.dumps(jobspec)) +time.sleep(2) + +#%% +# Finally, to watch (or stream) output, you can do the following. +# Each line here is a json structure that you can further parse. +# As an example, if "data" is present as a key, this usually is output +for line in flux.job.event_watch(handle, jobid, "guest.output"): + print(line) + + + diff --git a/tutorials/commands/flux-jobs.rst b/tutorials/commands/flux-jobs.rst index d14372b0..41390936 100644 --- a/tutorials/commands/flux-jobs.rst +++ b/tutorials/commands/flux-jobs.rst @@ -302,6 +302,7 @@ By default, ``flux jobs`` will not list jobs that are running under subinstances this with an example. Submit the following script to ``flux batch``. .. code-block:: sh + #!/bin/sh # filename: batchjob.sh