From c0192c8846f11d3e08fc9eb93d5ffaf2fbb34c28 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Thu, 13 Dec 2018 20:09:32 -0600 Subject: [PATCH 01/14] Post basic Octave support. --- course/page/__init__.py | 2 + course/page/code.py | 943 ++++++++++++++++++- course/page/code_runoc_backend.py | 314 ++++++ docker-image-run-octave/Dockerfile | 57 ++ docker-image-run-octave/docker-build.sh | 4 + docker-image-run-octave/flatten-container.sh | 12 + docker-image-run-octave/runoc | 156 +++ docker-image-run-py/docker-build.sh | 2 +- 8 files changed, 1488 insertions(+), 2 deletions(-) create mode 100644 course/page/code_runoc_backend.py create mode 100644 docker-image-run-octave/Dockerfile create mode 100755 docker-image-run-octave/docker-build.sh create mode 100755 docker-image-run-octave/flatten-container.sh create mode 100755 docker-image-run-octave/runoc diff --git a/course/page/__init__.py b/course/page/__init__.py index e268f9d69..39c9ce5ce 100644 --- a/course/page/__init__.py +++ b/course/page/__init__.py @@ -37,6 +37,8 @@ ChoiceQuestion, MultipleChoiceQuestion, SurveyChoiceQuestion) from course.page.code import ( PythonCodeQuestion, PythonCodeQuestionWithHumanTextFeedback) +from course.page.code import ( + OctaveCodeQuestion) from course.page.upload import FileUploadQuestion __all__ = ( diff --git a/course/page/code.py b/course/page/code.py index 96a878a4c..d95ff2faa 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -46,6 +46,7 @@ # True for 'spawn containers' (normal operation) # False for 'just connect to localhost:RUNPY_PORT' for runpy' SPAWN_CONTAINERS_FOR_RUNPY = True +SPAWN_CONTAINERS_FOR_RUNOC = True # {{{ python code question @@ -260,7 +261,6 @@ def check_timeout(): # Oh well. No need to bother the students with this nonsense. pass - def is_nuisance_failure(result): if result["result"] != "uncaught_error": return False @@ -1190,4 +1190,945 @@ def grade(self, page_context, page_data, answer_data, grade_data): # }}} +# {{{ octave code question + +class OctaveCodeForm(StyledForm): + # prevents form submission with codemirror's empty textarea + use_required_attribute = False + + def __init__(self, read_only, interaction_mode, initial_code, + data=None, *args, **kwargs): + super(OctaveCodeForm, self).__init__(data, *args, **kwargs) #TODO OctaveCodeForm + + from course.utils import get_codemirror_widget + cm_widget, cm_help_text = get_codemirror_widget( + language_mode="octave", + interaction_mode=interaction_mode, + read_only=read_only, + + # Automatically focus the text field once there has + # been some input. + autofocus=( + not read_only + and (data is not None and "answer" in data))) + + self.fields["answer"] = forms.CharField(required=True, + initial=initial_code, + help_text=cm_help_text, + widget=cm_widget, + label=_("Answer")) + + self.style_codemirror_widget() + + def clean(self): + # FIXME Should try compilation + pass + + +RUNOC_PORT = 9942 +DOCKER_TIMEOUT = 15 + + +class InvalidPingResponse(RuntimeError): + pass + + +def request_octave_run(run_req, run_timeout, image=None): + import json + from six.moves import http_client + import docker + import socket + import errno + from docker.errors import APIError as DockerAPIError + + debug = True + if debug: + def debug_print(s): + print(s) + else: + def debug_print(s): + pass + + if SPAWN_CONTAINERS_FOR_RUNOC: + docker_url = getattr(settings, "RELATE_DOCKER_URL", + "unix://var/run/docker.sock") + docker_tls = getattr(settings, "RELATE_DOCKER_TLS_CONFIG", + None) + docker_cnx = docker.Client( + base_url=docker_url, + tls=docker_tls, + timeout=DOCKER_TIMEOUT, + version="1.19") + + if image is None: + image = settings.RELATE_DOCKER_RUNOC_IMAGE + + dresult = docker_cnx.create_container( + image=image, + command=[ + "/opt/runoc/runoc", + "-1"], + host_config={ + "Memory": 384*10**6, + "MemorySwap": -1, + "PublishAllPorts": True, + # Do not enable: matplotlib stops working if enabled. + # "ReadonlyRootfs": True, + }, + user="runoc") + + container_id = dresult["Id"] + else: + container_id = None + + connect_host_ip = 'localhost' + + try: + # FIXME: Prohibit networking + + if container_id is not None: + docker_cnx.start(container_id) + + container_props = docker_cnx.inspect_container(container_id) + (port_info,) = (container_props + ["NetworkSettings"]["Ports"]["%d/tcp" % RUNOC_PORT]) + port_host_ip = port_info.get("HostIp") + + if port_host_ip != "0.0.0.0": + connect_host_ip = port_host_ip + + port = int(port_info["HostPort"]) + else: + port = RUNOC_PORT + + from time import time, sleep + start_time = time() + + # {{{ ping until response received + + from traceback import format_exc + + def check_timeout(): + if time() - start_time < DOCKER_TIMEOUT: + sleep(0.1) + # and retry + else: + return { + "result": "uncaught_error", + "message": "Timeout waiting for container.", + "traceback": "".join(format_exc()), + "exec_host": connect_host_ip, + } + + while True: + try: + connection = http_client.HTTPConnection(connect_host_ip, port) + + connection.request('GET', '/ping') + + response = connection.getresponse() + response_data = response.read().decode() + + if response_data != "OK": + raise InvalidPingResponse() + + break + + except (http_client.BadStatusLine, InvalidPingResponse): + ct_res = check_timeout() + if ct_res is not None: + return ct_res + + except socket.error as e: + if e.errno in [errno.ECONNRESET, errno.ECONNREFUSED]: + ct_res = check_timeout() + if ct_res is not None: + return ct_res + + else: + raise + + # }}} + + debug_print("PING SUCCESSFUL") + + try: + # Add a second to accommodate 'wire' delays + connection = http_client.HTTPConnection(connect_host_ip, port, + timeout=1 + run_timeout) + + headers = {'Content-type': 'application/json'} + + json_run_req = json.dumps(run_req).encode("utf-8") + + from time import time + start_time = time() + + debug_print("BEFPOST") + connection.request('POST', '/run-octave', json_run_req, headers) + debug_print("AFTPOST") + + http_response = connection.getresponse() + debug_print("GETR") + response_data = http_response.read().decode("utf-8") + debug_print("READR") + + end_time = time() + + result = json.loads(response_data) + + result["feedback"] = (result.get("feedback", []) + + ["Execution time: %.1f s -- Time limit: %.1f s" + % (end_time - start_time, run_timeout)]) + + result["exec_host"] = connect_host_ip + + return result + + except socket.timeout: + return { + "result": "timeout", + "exec_host": connect_host_ip, + } + finally: + if container_id is not None: + debug_print("-----------BEGIN DOCKER LOGS for %s" % container_id) + debug_print(docker_cnx.logs(container_id)) + debug_print("-----------END DOCKER LOGS for %s" % container_id) + + try: + docker_cnx.remove_container(container_id, force=True) + except DockerAPIError: + # Oh well. No need to bother the students with this nonsense. + pass + + +def is_nuisance_failure(result): + if result["result"] != "uncaught_error": + return False + + if "traceback" in result: + if "BadStatusLine" in result["traceback"]: + + # Occasionally, we fail to send a POST to the container, even after + # the inital ping GET succeeded, for (for now) mysterious reasons. + # Just try again. + + return True + + if "bind: address already in use" in result["traceback"]: + # https://github.com/docker/docker/issues/8714 + + return True + + if ("requests.packages.urllib3.exceptions.NewConnectionError" + in result["traceback"]): + return True + + if "http.client.RemoteDisconnected" in result["traceback"]: + return True + + if "[Errno 113] No route to host" in result["traceback"]: + return True + + return False + + +def request_octave_run_with_retries(run_req, run_timeout, image=None, retry_count=3): + while True: + result = request_octave_run(run_req, run_timeout, image=image) + + if retry_count and is_nuisance_failure(result): + retry_count -= 1 + continue + + return result + + +class OctaveCodeQuestion(PageBaseWithTitle, PageBaseWithValue): + """ + An auto-graded question allowing an answer consisting of Octave code. + All user code as well as all code specified as part of the problem + is in Octave 4.2+. + + If you are not including the + :attr:`course.constants.flow_permission.change_answer` + permission for your entire flow, you likely want to + include this snippet in your question definition: + + .. code-block:: yaml + + access_rules: + add_permissions: + - change_answer + + This will allow participants multiple attempts at getting + the right answer. + + .. attribute:: id + + |id-page-attr| + + .. attribute:: type + + ``OctaveCodeQuestion`` + + .. attribute:: is_optional_page + + |is-optional-page-attr| + + .. attribute:: access_rules + + |access-rules-page-attr| + + .. attribute:: title + + |title-page-attr| + + .. attribute:: value + + |value-page-attr| + + .. attribute:: prompt + + The page's prompt, written in :ref:`markup`. + + .. attribute:: timeout + + A number, giving the number of seconds for which setup code, + the given answer code, and the test code (combined) will be + allowed to run. + + .. attribute:: setup_code + + Optional. + Octave code to prepare the environment for the participants + answer. + + .. attribute:: show_setup_code + + Optional. ``True`` or ``False``. If true, the :attr:`setup_code` + will be shown to the participant. + + .. attribute:: names_for_user + + Optional. + Symbols defined at the end of the :attr:`setup_code` that will be + made available to the participant's code. + + A deep copy (using the standard library function :func:`copy.deepcopy`) + of these values is made, to prevent the user from modifying trusted + state of the grading code. + + .. attribute:: names_from_user + + Optional. + Symbols that the participant's code is expected to define. + These will be made available to the :attr:`test_code`. + Some remapping of types will be made between Octave and Python classes. + + .. attribute:: test_code + + Optional. + Code that will be run to determine the correctness of a + student-provided solution. Will have access to variables in + :attr:`names_from_user` (which will be *None*) if not provided. Should + never raise an exception. + + This may contain the marker "###CORRECT_CODE###", which will + be replaced with the contents of :attr:`correct_code`, with + each line indented to the same depth as where the marker + is found. The line with this marker is only allowed to have + white space and the marker on it. + + .. attribute:: show_test_code + + Optional. ``True`` or ``False``. If true, the :attr:`test_code` + will be shown to the participant. + + .. attribute:: correct_code_explanation + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). This is shown before + :attr:`correct_code` as an explanation. + + .. attribute:: correct_code + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). + + .. attribute:: initial_code + + Optional. + Code present in the code input field when the participant first starts + working on their solution. + + .. attribute:: data_files + + Optional. + A list of file names in the :ref:`git-repo` whose contents will be made + available to :attr:`setup_code` and :attr:`test_code` through the + ``data_files`` dictionary. (see below) + + .. attribute:: single_submission + + Optional, a Boolean. If the question does not allow multiple submissions + based on its :attr:`access_rules` (not the ones of the flow), a warning + is shown. Setting this attribute to True will silence the warning. + + The following symbols are available in :attr:`setup_code` and :attr:`test_code`: + + * ``GradingComplete``: An exception class that can be raised to indicated + that the grading code has concluded. + + * ``feedback``: A class instance with the following interface:: + + feedback.set_points(0.5) # 0<=points<=1 (usually) + feedback.add_feedback("This was wrong") + + # combines the above two and raises GradingComplete + feedback.finish(0, "This was wrong") + + feedback.check_numpy_array_sanity(name, num_axes, data) + + feedback.check_numpy_array_features(name, ref, data, report_failure=True) + + feedback.check_numpy_array_allclose(name, ref, data, + accuracy_critical=True, rtol=1e-5, atol=1e-8, + report_success=True, report_failure=True) + # If report_failure is True, this function will only return + # if *data* passes the tests. It will return *True* in this + # case. + # + # If report_failure is False, this function will always return, + # and the return value will indicate whether *data* passed the + # accuracy/shape/kind checks. + + feedback.check_list(name, ref, data, entry_type=None) + + feedback.check_scalar(name, ref, data, accuracy_critical=True, + rtol=1e-5, atol=1e-8, report_success=True, report_failure=True) + # returns True if accurate + + feedback.call_user(f, *args, **kwargs) + # Calls a user-supplied function and prints an appropriate + # feedback message in case of failure. + + * ``data_files``: A dictionary mapping file names from :attr:`data_files` + to :class:`bytes` instances with that file's contents. + + * ``user_code``: The user code being tested, as a string. + """ + + def __init__(self, vctx, location, page_desc): + super(OctaveCodeQuestion, self).__init__(vctx, location, page_desc) + + if vctx is not None and hasattr(page_desc, "data_files"): + for data_file in page_desc.data_files: + try: + if not isinstance(data_file, str): + raise ObjectDoesNotExist() + + from course.content import get_repo_blob + get_repo_blob(vctx.repo, data_file, vctx.commit_sha) + except ObjectDoesNotExist: + raise ValidationError( + string_concat( + "%(location)s: ", + _("data file '%(file)s' not found")) + % {"location": location, "file": data_file}) + + if not getattr(page_desc, "single_submission", False) and vctx is not None: + is_multi_submit = False + + if hasattr(page_desc, "access_rules"): + if hasattr(page_desc.access_rules, "add_permissions"): + if (flow_permission.change_answer + in page_desc.access_rules.add_permissions): + is_multi_submit = True + + if not is_multi_submit: + vctx.add_warning(location, _("code question does not explicitly " + "allow multiple submission. Either add " + "access_rules/add_permssions/change_answer " + "or add 'single_submission: True' to confirm that you intend " + "for only a single submission to be allowed. " + "While you're at it, consider adding " + "access_rules/add_permssions/see_correctness.")) + + def required_attrs(self): + return super(OctaveCodeQuestion, self).required_attrs() + ( + ("prompt", "markup"), + ("timeout", (int, float)), + ) + + def allowed_attrs(self): + return super(OctaveCodeQuestion, self).allowed_attrs() + ( + ("setup_code", str), + ("show_setup_code", bool), + ("names_for_user", list), + ("names_from_user", list), + ("test_code", str), + ("show_test_code", bool), + ("correct_code_explanation", "markup"), + ("correct_code", str), + ("initial_code", str), + ("data_files", list), + ("single_submission", bool), + ) + + def _initial_code(self): + result = getattr(self.page_desc, "initial_code", None) + if result is not None: + return result.strip() + else: + return result + + def markup_body_for_title(self): + return self.page_desc.prompt + + def body(self, page_context, page_data): + from django.template.loader import render_to_string + return render_to_string( + "course/prompt-code-question.html", + { + "prompt_html": + markup_to_html(page_context, self.page_desc.prompt), + "initial_code": self._initial_code(), + "show_setup_code": getattr( + self.page_desc, "show_setup_code", False), + "setup_code": getattr(self.page_desc, "setup_code", ""), + "show_test_code": getattr( + self.page_desc, "show_test_code", False), + "test_code": getattr(self.page_desc, "test_code", ""), + }) + + def make_form(self, page_context, page_data, + answer_data, page_behavior): + + if answer_data is not None: + answer = {"answer": answer_data["answer"]} + form = OctaveCodeForm( + not page_behavior.may_change_answer, + get_editor_interaction_mode(page_context), + self._initial_code(), + answer) + else: + answer = None + form = OctaveCodeForm( + not page_behavior.may_change_answer, + get_editor_interaction_mode(page_context), + self._initial_code(), + ) + + return form + + def process_form_post( + self, page_context, page_data, post_data, files_data, page_behavior): + return OctaveCodeForm( + not page_behavior.may_change_answer, + get_editor_interaction_mode(page_context), + self._initial_code(), + post_data, files_data) + + def answer_data(self, page_context, page_data, form, files_data): + return {"answer": form.cleaned_data["answer"].strip()} + + def get_test_code(self): + test_code = getattr(self.page_desc, "test_code", None) + if test_code is None: + return test_code + + correct_code = getattr(self.page_desc, "correct_code", None) + if correct_code is None: + correct_code = "" + + from .code_runoc_backend import substitute_correct_code_into_test_code + return substitute_correct_code_into_test_code(test_code, correct_code) + + def grade(self, page_context, page_data, answer_data, grade_data): + if answer_data is None: + return AnswerFeedback(correctness=0, + feedback=_("No answer provided.")) + + user_code = answer_data["answer"] + + # {{{ request run + + run_req = {"compile_only": False, "user_code": user_code} + + def transfer_attr(name): + if hasattr(self.page_desc, name): + run_req[name] = getattr(self.page_desc, name) + + transfer_attr("setup_code") + transfer_attr("names_for_user") + transfer_attr("names_from_user") + + run_req["test_code"] = self.get_test_code() + + if hasattr(self.page_desc, "data_files"): + run_req["data_files"] = {} + + from course.content import get_repo_blob + + # TODO: set this up to work with M-files for Octave functions + for data_file in self.page_desc.data_files: + from base64 import b64encode + run_req["data_files"][data_file] = \ + b64encode( + get_repo_blob( + page_context.repo, data_file, + page_context.commit_sha).data).decode() + + try: + response_dict = request_octave_run_with_retries(run_req, + run_timeout=self.page_desc.timeout) + except Exception: + from traceback import format_exc + response_dict = { + "result": "uncaught_error", + "message": "Error connecting to container", + "traceback": "".join(format_exc()), + } + + # }}} + + feedback_bits = [] + + correctness = None + + if "points" in response_dict: + correctness = response_dict["points"] + try: + feedback_bits.append( + "

%s

" + % _(get_auto_feedback(correctness))) + except Exception as e: + correctness = None + response_dict["result"] = "setup_error" + response_dict["message"] = ( + "%s: %s" % (type(e).__name__, str(e)) + ) + + # {{{ send email if the grading code broke + + if response_dict["result"] in [ + "uncaught_error", + "setup_compile_error", + "setup_error", + "test_compile_error", + "test_error"]: + error_msg_parts = ["RESULT: %s" % response_dict["result"]] + for key, val in sorted(response_dict.items()): + if (key not in ["result", "figures"] + and val + and isinstance(val, six.string_types)): + error_msg_parts.append("-------------------------------------") + error_msg_parts.append(key) + error_msg_parts.append("-------------------------------------") + error_msg_parts.append(val) + error_msg_parts.append("-------------------------------------") + error_msg_parts.append("user code") + error_msg_parts.append("-------------------------------------") + error_msg_parts.append(user_code) + error_msg_parts.append("-------------------------------------") + + error_msg = "\n".join(error_msg_parts) + + from relate.utils import local_now, format_datetime_local + from course.utils import LanguageOverride + with LanguageOverride(page_context.course): + from relate.utils import render_email_template + message = render_email_template( + "course/broken-code-question-email.txt", { + "site": getattr(settings, "RELATE_BASE_URL"), + "page_id": self.page_desc.id, + "course": page_context.course, + "error_message": error_msg, + "review_uri": page_context.page_uri, + "time": format_datetime_local(local_now()) + }) + + if ( + not page_context.in_sandbox + and not is_nuisance_failure(response_dict)): + try: + from django.core.mail import EmailMessage + msg = EmailMessage("".join(["[%s:%s] ", + _("code question execution failed")]) + % ( + page_context.course.identifier, + page_context.flow_session.flow_id + if page_context.flow_session is not None + else _("")), + message, + settings.ROBOT_EMAIL_FROM, + [page_context.course.notify_email]) + + from relate.utils import get_outbound_mail_connection + msg.connection = get_outbound_mail_connection("robot") + msg.send() + + except Exception: + from traceback import format_exc + feedback_bits.append( + six.text_type(string_concat( + "

", + _( + "Both the grading code and the attempt to " + "notify course staff about the issue failed. " + "Please contact the course or site staff and " + "inform them of this issue, mentioning this " + "entire error message:"), + "

", + "

", + _( + "Sending an email to the course staff about the " + "following failure failed with " + "the following error message:"), + "

",
+                                "".join(format_exc()),
+                                "
", + _("The original failure message follows:"), + "

"))) + + # }}} + + if hasattr(self.page_desc, "correct_code"): + def normalize_code(s): + return (s + .replace(" ", "") + .replace("\r", "") + .replace("\n", "") + .replace("\t", "")) + + if (normalize_code(user_code) + == normalize_code(self.page_desc.correct_code)): + feedback_bits.append( + "

%s

" + % _("It looks like you submitted code that is identical to " + "the reference solution. This is not allowed.")) + + from relate.utils import dict_to_struct + response = dict_to_struct(response_dict) + + bulk_feedback_bits = [] + + if response.result == "success": + pass + elif response.result in [ + "uncaught_error", + "setup_compile_error", + "setup_error", + "test_compile_error", + "test_error"]: + feedback_bits.append("".join([ + "

", + _( + "The grading code failed. Sorry about that. " + "The staff has been informed, and if this problem is " + "due to an issue with the grading code, " + "it will be fixed as soon as possible. " + "In the meantime, you'll see a traceback " + "below that may help you figure out what went wrong." + ), + "

"])) + elif response.result == "timeout": + feedback_bits.append("".join([ + "

", + _( + "Your code took too long to execute. The problem " + "specifies that your code may take at most %s seconds " + "to run. " + "It took longer than that and was aborted." + ), + "

"]) + % self.page_desc.timeout) + + correctness = 0 + elif response.result == "user_compile_error": + feedback_bits.append("".join([ + "

", + _("Your code failed to compile. An error message is " + "below."), + "

"])) + + correctness = 0 + elif response.result == "user_error": + feedback_bits.append("".join([ + "

", + _("Your code failed with an exception. " + "A traceback is below."), + "

"])) + + correctness = 0 + else: + raise RuntimeError("invalid runoc result: %s" % response.result) + + if hasattr(response, "feedback") and response.feedback: + def sanitize(s): + import bleach + return bleach.clean(s, tags=["p", "pre"]) + feedback_bits.append("".join([ + "

", + _("Here is some feedback on your code"), + ":" + "

"]) % + "".join( + "
  • %s
    • " % sanitize(fb_item) + for fb_item in response.feedback)) + if hasattr(response, "traceback") and response.traceback: + feedback_bits.append("".join([ + "

    ", + _("This is the exception traceback"), + ":" + "

    %s

    "]) % escape(response.traceback)) + if hasattr(response, "exec_host") and response.exec_host != "localhost": + import socket + try: + exec_host_name, dummy, dummy = socket.gethostbyaddr( + response.exec_host) + except socket.error: + exec_host_name = response.exec_host + + feedback_bits.append("".join([ + "

    ", + _("Your code ran on %s.") % exec_host_name, + "

    "])) + + if hasattr(response, "stdout") and response.stdout: + bulk_feedback_bits.append("".join([ + "

    ", + _("Your code printed the following output"), + ":" + "

    %s

    "]) + % escape(response.stdout)) + if hasattr(response, "stderr") and response.stderr: + bulk_feedback_bits.append("".join([ + "

    ", + _("Your code printed the following error messages"), + ":" + "

    %s

    "]) % escape(response.stderr)) + if hasattr(response, "figures") and response.figures: + fig_lines = ["".join([ + "

    ", + _("Your code produced the following plots"), + ":

    "]), + '
    ', + ] + + for nr, mime_type, b64data in response.figures: + if mime_type in ["image/jpeg", "image/png"]: + fig_lines.extend([ + "".join([ + "
    ", + _("Figure"), "%d
    "]) % nr, + '
    Figure %d
    ' + % (nr, mime_type, b64data)]) + + fig_lines.append("
    ") + bulk_feedback_bits.extend(fig_lines) + + # {{{ html output / santization + + if hasattr(response, "html") and response.html: + def is_allowed_data_uri(allowed_mimetypes, uri): + import re + m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri) + if not m: + return False + + mimetype = m.group(1) + return mimetype in allowed_mimetypes + + def sanitize(s): + import bleach + + def filter_audio_attributes(tag, name, value): + if name in ["controls"]: + return True + else: + return False + + def filter_source_attributes(tag, name, value): + if name in ["type"]: + return True + elif name == "src": + if is_allowed_data_uri([ + "audio/wav", + ], value): + return bleach.sanitizer.VALUE_SAFE + else: + return False + else: + return False + + def filter_img_attributes(tag, name, value): + if name in ["alt", "title"]: + return True + elif name == "src": + return is_allowed_data_uri([ + "image/png", + "image/jpeg", + ], value) + else: + return False + + if not isinstance(s, six.text_type): + return _("(Non-string in 'HTML' output filtered out)") + + return bleach.clean(s, + tags=bleach.ALLOWED_TAGS + ["audio", "video", "source"], + attributes={ + "audio": filter_audio_attributes, + "source": filter_source_attributes, + "img": filter_img_attributes, + }) + + bulk_feedback_bits.extend( + sanitize(snippet) for snippet in response.html) + + # }}} + + return AnswerFeedback( + correctness=correctness, + feedback="\n".join(feedback_bits), + bulk_feedback="\n".join(bulk_feedback_bits)) + + def correct_answer(self, page_context, page_data, answer_data, grade_data): + result = "" + + if hasattr(self.page_desc, "correct_code_explanation"): + result += markup_to_html( + page_context, + self.page_desc.correct_code_explanation) + + if hasattr(self.page_desc, "correct_code"): + result += ("".join([ + _("The following code is a valid answer"), + ": 
    %s
    "]) + % escape(self.page_desc.correct_code)) + + return result + + def normalized_answer(self, page_context, page_data, answer_data): + if answer_data is None: + return None + + normalized_answer = answer_data["answer"] + + from django.utils.html import escape + return "
    %s
    " % escape(normalized_answer) + + def normalized_bytes_answer(self, page_context, page_data, answer_data): + if answer_data is None: + return None + + return (".m", answer_data["answer"].encode("utf-8")) + +# }}} + # vim: foldmethod=marker diff --git a/course/page/code_runoc_backend.py b/course/page/code_runoc_backend.py new file mode 100644 index 000000000..0c1499423 --- /dev/null +++ b/course/page/code_runoc_backend.py @@ -0,0 +1,314 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import sys +import traceback + +try: + from .code_feedback import Feedback, GradingComplete +except SystemError: + from code_feedback import Feedback, GradingComplete # type: ignore +except ImportError: + from code_feedback import Feedback, GradingComplete # type: ignore + + +__doc__ = """ +PROTOCOL +======== + +.. class:: Request + + .. attribute:: setup_code + + .. attribute:: names_for_user + + .. attribute:: user_code + + .. attribute:: names_from_user + + .. attribute:: test_code + + .. attribute:: data_files + + A dictionary from data file names to their + base64-cencoded contents. + Optional. + + .. attribute:: compile_only + + :class:`bool` + +.. class Response:: + .. attribute:: result + + One of + + * ``success`` + * ``timeout`` + * ``uncaught_error`` + * ``setup_compile_error`` + * ``setup_error``, + * ``user_compile_error`` + * ``user_error`` + * ``test_compile_error`` + * ``test_error`` + + Always present. + + .. attribute:: message + + Optional. + + .. attribute:: traceback + + Optional. + + .. attribute:: stdout + + Whatever came out of stdout. + + Optional. + + .. attribute:: stderr + + Whatever came out of stderr. + + Optional. + + .. attribute:: figures + + A list of ``(index, mime_type, string)``, where *string* is a + base64-encoded representation of the figure. *index* will usually + correspond to the matplotlib figure number. + + Optional. + + .. attribute:: html + + A list of HTML strings generated. These are aggressively sanitized + before being rendered. + + .. attribute:: points + + A number between 0 and 1 (inclusive). + + Present on ``success`` if :attr:`Request.compile_only` is *False*. + + .. attribute:: feedback + + A list of strings. + + Present on ``success`` if :attr:`Request.compile_only` is *False*. +""" + + +# {{{ tools + +class Struct(object): + def __init__(self, entries): + for name, val in entries.items(): + self.__dict__[name] = val + + def __repr__(self): + return repr(self.__dict__) + +# }}} + + +def substitute_correct_code_into_test_code(test_code, correct_code): + import re + CORRECT_CODE_TAG = re.compile(r"^(\s*)###CORRECT_CODE###\s*$") # noqa + + new_test_code_lines = [] + for l in test_code.split("\n"): + match = CORRECT_CODE_TAG.match(l) + if match is not None: + prefix = match.group(1) + for cc_l in correct_code.split("\n"): + new_test_code_lines.append(prefix+cc_l) + else: + new_test_code_lines.append(l) + + return "\n".join(new_test_code_lines) + + +def package_exception(result, what): + tp, val, tb = sys.exc_info() + result["result"] = what + result["message"] = "%s: %s" % (tp.__name__, str(val)) + result["traceback"] = "".join( + traceback.format_exception(tp, val, tb)) + + +def run_code(result, run_req): + # {{{ silence matplotlib font cache warnings + + import warnings + warnings.filterwarnings( + "ignore", message="Matplotlib is building the font cache.*") + + # }}} + + # {{{ compile code + + if getattr(run_req, "setup_code", None): + try: + setup_code = compile( + run_req.setup_code, "[setup code]", 'exec') + except Exception: + package_exception(result, "setup_compile_error") + return + else: + setup_code = None + + try: + user_code = compile( + run_req.user_code, "[user code]", 'exec') + except Exception: + package_exception(result, "user_compile_error") + return + + if getattr(run_req, "test_code", None): + try: + test_code = compile( + run_req.test_code, "[test code]", 'exec') + except Exception: + package_exception(result, "test_compile_error") + return + else: + test_code = None + + # }}} + + if hasattr(run_req, "compile_only") and run_req.compile_only: + result["result"] = "success" + return + + # {{{ run code + + data_files = {} + if hasattr(run_req, "data_files"): + from base64 import b64decode + for name, contents in run_req.data_files.items(): + data_files[name] = b64decode(contents.encode()) + + generated_html = [] + result["html"] = generated_html + + def output_html(s): + generated_html.append(s) + + feedback = Feedback() + maint_ctx = { + "feedback": feedback, + "user_code": run_req.user_code, + "data_files": data_files, + "output_html": output_html, + "GradingComplete": GradingComplete, + } + + if setup_code is not None: + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.setup_code + exec(setup_code, maint_ctx) + except Exception: + package_exception(result, "setup_error") + return + + user_ctx = {} + if hasattr(run_req, "names_for_user"): + for name in run_req.names_for_user: + if name not in maint_ctx: + result["result"] = "setup_error" + result["message"] = "Setup code did not define '%s'." % name + + user_ctx[name] = maint_ctx[name] + + from copy import deepcopy + user_ctx = deepcopy(user_ctx) + + try: + user_ctx["_MODULE_SOURCE_CODE"] = run_req.user_code + exec(user_code, user_ctx) + except Exception: + package_exception(result, "user_error") + return + + # {{{ export plots + + ''' + if "matplotlib" in sys.modules: + import matplotlib.pyplot as pt + from io import BytesIO + from base64 import b64encode + + format = "png" + mime = "image/png" + figures = [] + + for fignum in pt.get_fignums(): + pt.figure(fignum) + bio = BytesIO() + try: + pt.savefig(bio, format=format) + except Exception: + pass + else: + figures.append( + (fignum, mime, b64encode(bio.getvalue()).decode())) + + result["figures"] = figures + ''' + # }}} + + if hasattr(run_req, "names_from_user"): + for name in run_req.names_from_user: + if name not in user_ctx: + feedback.add_feedback( + "Required answer variable '%s' is not defined." + % name) + maint_ctx[name] = None + else: + maint_ctx[name] = user_ctx[name] + + if test_code is not None: + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.test_code + exec(test_code, maint_ctx) + except GradingComplete: + pass + except Exception: + package_exception(result, "test_error") + return + + result["points"] = feedback.points + result["feedback"] = feedback.feedback_items + + # }}} + + result["result"] = "success" + +# vim: foldmethod=marker diff --git a/docker-image-run-octave/Dockerfile b/docker-image-run-octave/Dockerfile new file mode 100644 index 000000000..673e11d12 --- /dev/null +++ b/docker-image-run-octave/Dockerfile @@ -0,0 +1,57 @@ +FROM inducer/debian-amd64-minbase +MAINTAINER Neal Davis +EXPOSE 9942 +RUN useradd runoc + +# Based on `compdatasci/octave-desktop` Docker image + +# Temporarily needed for pandas + +ARG OCTAVE_VERSION=4.2.1 + +# Install system packages and Octave +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + wget \ + curl \ + build-essential \ + gfortran \ + cmake \ + bsdtar \ + rsync \ + imagemagick \ + \ + gnuplot-x11 \ + libopenblas-base \ + \ + octave \ + liboctave-dev \ + octave-info \ + octave-symbolic \ + octave-parallel \ + octave-struct \ + \ + python3-dev \ + pandoc \ + ttf-dejavu && \ + apt-get clean && \ + apt-get autoremove && \ + curl -L https://github.com/hbin/top-programming-fonts/raw/master/install.sh | bash && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN apt-get clean +RUN fc-cache + +RUN mkdir -p /opt/runoc +ADD runoc /opt/runoc/ +COPY code_feedback.py /opt/runoc/ +COPY code_runoc_backend.py /opt/runoc/ + +# currently no graphics support +#RUN sed -i s/TkAgg/Agg/ /etc/matplotlibrc +#RUN echo "savefig.dpi : 80" >> /etc/matplotlibrc +#RUN echo "savefig.bbox : tight" >> /etc/matplotlibrc + +RUN rm -Rf /root/.cache + +# may use ./flatten-container.sh to reduce disk space diff --git a/docker-image-run-octave/docker-build.sh b/docker-image-run-octave/docker-build.sh new file mode 100755 index 000000000..eb04820fa --- /dev/null +++ b/docker-image-run-octave/docker-build.sh @@ -0,0 +1,4 @@ +#! /bin/sh +cp ../course/page/code_feedback.py ../course/page/code_runoc_backend.py . +docker build --no-cache . -t inducer/runoc +rm code_feedback.py code_runoc_backend.py diff --git a/docker-image-run-octave/flatten-container.sh b/docker-image-run-octave/flatten-container.sh new file mode 100755 index 000000000..d18bb6434 --- /dev/null +++ b/docker-image-run-octave/flatten-container.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +if test "$1" = ""; then + echo "$0 imagename" + exit 1 +fi +CONTAINER=$(docker create "$1") +docker export "$CONTAINER" | \ + docker import \ + -c "EXPOSE 9941" \ + - +docker rm -f $CONTAINER diff --git a/docker-image-run-octave/runoc b/docker-image-run-octave/runoc new file mode 100755 index 000000000..3a5f6670f --- /dev/null +++ b/docker-image-run-octave/runoc @@ -0,0 +1,156 @@ +#! /usr/bin/env python3 + +# placate flake8 +from __future__ import print_function + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import socketserver +import json +import sys +import io +try: + from code_runoc_backend import Struct, run_code, package_exception +except ImportError: + try: + # When faking a container for unittest + from course.page.code_runoc_backend import ( + Struct, run_code, package_exception) + except ImportError: + # When debugging, i.e., run "python runpy" command line + import os + sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir))) + from course.page.code_runoc_backend import ( + Struct, run_code, package_exception) + +from http.server import BaseHTTPRequestHandler + +PORT = 9942 +OUTPUT_LENGTH_LIMIT = 16*1024 + +TEST_COUNT = 0 + + +def truncate_if_long(s): + if len(s) > OUTPUT_LENGTH_LIMIT: + s = s[:OUTPUT_LENGTH_LIMIT] + "[TRUNCATED... TOO MUCH OUTPUT]" + return s + + +class RunRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + print("GET RECEIVED", file=sys.stderr) + if self.path != "/ping": + raise RuntimeError("unrecognized path in GET") + + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + + self.wfile.write(b"OK") + print("PING RESPONSE DONE", file=sys.stderr) + + def do_POST(self): + global TEST_COUNT + TEST_COUNT += 1 + + response = {} + + prev_stdout = sys.stdout # noqa + prev_stderr = sys.stderr # noqa + + try: + print("POST RECEIVED", file=prev_stderr) + if self.path != "/run-octave": + raise RuntimeError("unrecognized path in POST") + + clength = int(self.headers['content-length']) + recv_data = self.rfile.read(clength) + + print("RUNOC RECEIVED %d bytes" % len(recv_data), + file=prev_stderr) + run_req = Struct(json.loads(recv_data.decode("utf-8"))) + print("REQUEST: %r" % run_req, file=prev_stderr) + + stdout = io.StringIO() + stderr = io.StringIO() + + sys.stdin = None + sys.stdout = stdout + sys.stderr = stderr + + run_code(response, run_req) + + response["stdout"] = truncate_if_long(stdout.getvalue()) + response["stderr"] = truncate_if_long(stderr.getvalue()) + + print("REQUEST SERVICED: %r" % response, file=prev_stderr) + + json_result = json.dumps(response).encode("utf-8") + + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + + print("WRITING RESPONSE", file=prev_stderr) + self.wfile.write(json_result) + print("WROTE RESPONSE", file=prev_stderr) + except: + print("ERROR RESPONSE", file=prev_stderr) + response = {} + package_exception(response, "uncaught_error") + json_result = json.dumps(response).encode("utf-8") + + self.send_response(500) + self.send_header("Content-type", "application/json") + self.end_headers() + + self.wfile.write(json_result) + finally: + sys.stdout = prev_stdout + sys.stderr = prev_stderr + + +def main(): + print("STARTING, LISTENING ON %d" % PORT, file=sys.stderr) + server = socketserver.TCPServer(("", PORT), RunRequestHandler) + + serve_single_test = len(sys.argv) > 1 and sys.argv[1] == "-1" + + while True: + server.handle_request() + print("SERVED REQUEST", file=sys.stderr) + if TEST_COUNT > 0 and serve_single_test: + break + + server.server_close() + print("FINISHED server_close()", file=sys.stderr) + + print("EXITING", file=sys.stderr) + + +if __name__ == "__main__": + main() + +# vim: foldmethod=marker diff --git a/docker-image-run-py/docker-build.sh b/docker-image-run-py/docker-build.sh index 3943a4e71..9f7b73bfc 100755 --- a/docker-image-run-py/docker-build.sh +++ b/docker-image-run-py/docker-build.sh @@ -1,5 +1,5 @@ #! /bin/sh cp ../course/page/code_feedback.py ../course/page/code_runpy_backend.py . -docker build --no-cache . +docker build --no-cache . -t inducer/relate-runpy-amd64 rm code_feedback.py code_runpy_backend.py From 50ca0d1d269ff4ad9000c388264277aa4952c314 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Sat, 15 Dec 2018 03:40:54 -0600 Subject: [PATCH 02/14] Use two steps to handle copying code over to Docker image. --- bin/relate | 0 course/page/code_runoc_backend.py | 67 ++-- docker-image-run-octave/Dockerfile | 9 +- .../runoc-update/Dockerfile | 14 + .../runoc-update/docker-build.sh | 4 + tests/test_pages/test_code.py | 286 ++++++++++++++++++ 6 files changed, 330 insertions(+), 50 deletions(-) mode change 100644 => 100755 bin/relate create mode 100644 docker-image-run-octave/runoc-update/Dockerfile create mode 100644 docker-image-run-octave/runoc-update/docker-build.sh diff --git a/bin/relate b/bin/relate old mode 100644 new mode 100755 diff --git a/course/page/code_runoc_backend.py b/course/page/code_runoc_backend.py index 0c1499423..525f7dff7 100644 --- a/course/page/code_runoc_backend.py +++ b/course/page/code_runoc_backend.py @@ -164,56 +164,24 @@ def package_exception(result, what): def run_code(result, run_req): - # {{{ silence matplotlib font cache warnings + # {{{ set up octave process - import warnings - warnings.filterwarnings( - "ignore", message="Matplotlib is building the font cache.*") + import oct2py - # }}} - - # {{{ compile code - - if getattr(run_req, "setup_code", None): - try: - setup_code = compile( - run_req.setup_code, "[setup code]", 'exec') - except Exception: - package_exception(result, "setup_compile_error") - return - else: - setup_code = None - - try: - user_code = compile( - run_req.user_code, "[user code]", 'exec') - except Exception: - package_exception(result, "user_compile_error") - return - - if getattr(run_req, "test_code", None): - try: - test_code = compile( - run_req.test_code, "[test code]", 'exec') - except Exception: - package_exception(result, "test_compile_error") - return - else: - test_code = None + oc = oct2py.Oct2Py() # }}} - if hasattr(run_req, "compile_only") and run_req.compile_only: - result["result"] = "success" - return - # {{{ run code data_files = {} if hasattr(run_req, "data_files"): from base64 import b64decode for name, contents in run_req.data_files.items(): + # This part "cheats" a litle, since Octave lets us evaluate functions + # in the same context as the main code. (MATLAB segregates these.) data_files[name] = b64decode(contents.encode()) + oc.eval(b64decode(contents.encode()).decode("utf-8")) generated_html = [] result["html"] = generated_html @@ -230,16 +198,16 @@ def output_html(s): "GradingComplete": GradingComplete, } - if setup_code is not None: + if run_req.setup_code is not None: try: - maint_ctx["_MODULE_SOURCE_CODE"] = run_req.setup_code - exec(setup_code, maint_ctx) + oc.eval(run_req.setup_code) except Exception: package_exception(result, "setup_error") return + ''' user_ctx = {} - if hasattr(run_req, "names_for_user"): + if hasattr(run_req, "names_for_user"): #XXX unused for Octave context currently for name in run_req.names_for_user: if name not in maint_ctx: result["result"] = "setup_error" @@ -249,10 +217,10 @@ def output_html(s): from copy import deepcopy user_ctx = deepcopy(user_ctx) + ''' try: - user_ctx["_MODULE_SOURCE_CODE"] = run_req.user_code - exec(user_code, user_ctx) + oc.eval(run_req.user_code) except Exception: package_exception(result, "user_error") return @@ -285,19 +253,20 @@ def output_html(s): # }}} if hasattr(run_req, "names_from_user"): + values = [] for name in run_req.names_from_user: - if name not in user_ctx: + try: + maint_ctx[name] = oc.pull(name) + except oct2py.Oct2PyError: feedback.add_feedback( "Required answer variable '%s' is not defined." % name) maint_ctx[name] = None - else: - maint_ctx[name] = user_ctx[name] - if test_code is not None: + if run_req.test_code is not None: # XXX test code is written in Python try: maint_ctx["_MODULE_SOURCE_CODE"] = run_req.test_code - exec(test_code, maint_ctx) + exec(run_req.test_code, maint_ctx) except GradingComplete: pass except Exception: diff --git a/docker-image-run-octave/Dockerfile b/docker-image-run-octave/Dockerfile index 673e11d12..6ce5f00c6 100644 --- a/docker-image-run-octave/Dockerfile +++ b/docker-image-run-octave/Dockerfile @@ -32,13 +32,20 @@ RUN apt-get update && \ octave-struct \ \ python3-dev \ + python3-setuptools \ + python3-pip \ + python3-numpy \ + python3-scipy \ + python3-matplotlib \ pandoc \ ttf-dejavu && \ apt-get clean && \ apt-get autoremove && \ - curl -L https://github.com/hbin/top-programming-fonts/raw/master/install.sh | bash && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +RUN apt-get update && \ + pip3 install oct2py + RUN apt-get clean RUN fc-cache diff --git a/docker-image-run-octave/runoc-update/Dockerfile b/docker-image-run-octave/runoc-update/Dockerfile new file mode 100644 index 000000000..634a3a567 --- /dev/null +++ b/docker-image-run-octave/runoc-update/Dockerfile @@ -0,0 +1,14 @@ +FROM inducer/runoc +MAINTAINER Neal Davis +EXPOSE 9942 +#RUN useradd runoc + +#RUN apt-get update && \ +# pip install oct2py + +RUN mkdir -p /opt/runoc +#ADD runoc /opt/runoc/ +COPY code_feedback.py /opt/runoc/ +COPY code_runoc_backend.py /opt/runoc/ + +RUN rm -Rf /root/.cache diff --git a/docker-image-run-octave/runoc-update/docker-build.sh b/docker-image-run-octave/runoc-update/docker-build.sh new file mode 100644 index 000000000..fb4e6ed9b --- /dev/null +++ b/docker-image-run-octave/runoc-update/docker-build.sh @@ -0,0 +1,4 @@ +#! /bin/sh +cp ../../course/page/code_feedback.py ../../course/page/code_runoc_backend.py . +docker build --no-cache . -t inducer/runoc-update +rm code_feedback.py code_runoc_backend.py diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index 5b22c1f53..374deefe6 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -35,6 +35,8 @@ from course.page.code import ( RUNPY_PORT, request_python_run_with_retries, InvalidPingResponse, is_nuisance_failure, PythonCodeQuestionWithHumanTextFeedback) +from course.page.code import ( + RUNOC_PORT, request_octave_run_with_retries) from course.utils import FlowPageContext, CoursePageContext from course.constants import MAX_EXTRA_CREDIT_FACTOR @@ -1284,6 +1286,290 @@ def test_docker_container_runpy_retries_count(self): self.assertEqual(mock_is_nuisance_failure.call_count, 1) +class RequestOctaveRunWithRetriesTest(unittest.TestCase): + # Testing course.page.code.request_octave_run_with_retries, + # adding tests for use cases that didn't cover in other tests + + @override_settings(RELATE_DOCKER_RUNOC_IMAGE="some_other_image") + def test_image_none(self): + # Testing if image is None, settings.RELATE_DOCKER_RUNPY_IMAGE is used + with mock.patch("docker.client.Client.create_container") as mock_create_ctn: + + # this will raise KeyError + mock_create_ctn.return_value = {} + + with self.assertRaises(KeyError): + request_python_run_with_retries( + run_req={}, run_timeout=0.1) + self.assertEqual(mock_create_ctn.call_count, 1) + self.assertIn("some_other_image", mock_create_ctn.call_args[0]) + + @override_settings(RELATE_DOCKER_RUNOC_IMAGE="some_other_image") + def test_image_not_none(self): + # Testing if image is None, settings.RELATE_DOCKER_RUNPY_IMAGE is used + with mock.patch("docker.client.Client.create_container") as mock_create_ctn: + + # this will raise KeyError + mock_create_ctn.return_value = {} + + my_image = "my_runoc_image" + + with self.assertRaises(KeyError): + request_python_run_with_retries( + run_req={}, image=my_image, run_timeout=0.1) + self.assertEqual(mock_create_ctn.call_count, 1) + self.assertIn(my_image, mock_create_ctn.call_args[0]) + + @skipIf(six.PY2, "PY2 doesn't support subTest") + def test_docker_container_ping_failure(self): + with ( + mock.patch("docker.client.Client.create_container")) as mock_create_ctn, ( # noqa + mock.patch("docker.client.Client.start")) as mock_ctn_start, ( + mock.patch("docker.client.Client.logs")) as mock_ctn_logs, ( + mock.patch("docker.client.Client.remove_container")) as mock_remove_ctn, ( # noqa + mock.patch("docker.client.Client.inspect_container")) as mock_inpect_ctn, ( # noqa + mock.patch("six.moves.http_client.HTTPConnection.request")) as mock_ctn_request: # noqa + + mock_create_ctn.return_value = {"Id": "someid"} + mock_ctn_start.side_effect = lambda x: None + mock_ctn_logs.side_effect = lambda x: None + mock_remove_ctn.return_value = None + fake_host_ip = "192.168.1.100" + fake_host_port = "69999" + + mock_inpect_ctn.return_value = { + "NetworkSettings": { + "Ports": {"%d/tcp" % RUNPY_PORT: ( + {"HostIp": fake_host_ip, "HostPort": fake_host_port}, + )} + }} + + with self.subTest(case="Docker ping timeout with BadStatusLine Error"): + from six.moves.http_client import BadStatusLine + fake_bad_statusline_msg = "my custom bad status" + mock_ctn_request.side_effect = BadStatusLine(fake_bad_statusline_msg) + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], fake_host_ip) + self.assertIn(fake_bad_statusline_msg, res["traceback"]) + + with self.subTest( + case="Docker ping timeout with InvalidPingResponse Error"): + invalid_ping_resp_msg = "my custom invalid ping response exception" + mock_ctn_request.side_effect = ( + InvalidPingResponse(invalid_ping_resp_msg)) + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], fake_host_ip) + self.assertIn(InvalidPingResponse.__name__, res["traceback"]) + self.assertIn(invalid_ping_resp_msg, res["traceback"]) + + with self.subTest( + case="Docker ping socket error with erron ECONNRESET"): + my_socket_error = socket_error() + my_socket_error.errno = errno.ECONNRESET + mock_ctn_request.side_effect = my_socket_error + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], fake_host_ip) + self.assertIn(type(my_socket_error).__name__, res["traceback"]) + + with self.subTest( + case="Docker ping socket error with erron ECONNREFUSED"): + my_socket_error = socket_error() + my_socket_error.errno = errno.ECONNREFUSED + mock_ctn_request.side_effect = my_socket_error + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], fake_host_ip) + self.assertIn(type(my_socket_error).__name__, res["traceback"]) + + with self.subTest( + case="Docker ping socket error with erron EAFNOSUPPORT"): + my_socket_error = socket_error() + + # This errno should raise error + my_socket_error.errno = errno.EAFNOSUPPORT + mock_ctn_request.side_effect = my_socket_error + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + with self.assertRaises(socket_error) as e: + request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(e.exception.errno, my_socket_error.errno) + + with self.assertRaises(socket_error) as e: + request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(e.exception.errno, my_socket_error.errno) + + # This should be the last subTest, because this will the behavior of + # change mock_remove_ctn + with self.subTest( + case="Docker ping timeout with InvalidPingResponse and " + "remove container failed with APIError"): + invalid_ping_resp_msg = "my custom invalid ping response exception" + fake_host_ip = "0.0.0.0" + + mock_inpect_ctn.return_value = { + "NetworkSettings": { + "Ports": {"%d/tcp" % RUNPY_PORT: ( + {"HostIp": fake_host_ip, "HostPort": fake_host_port}, + )} + }} + + mock_ctn_request.side_effect = ( + InvalidPingResponse(invalid_ping_resp_msg)) + mock_remove_ctn.reset_mock() + from django.http import HttpResponse + fake_response_content = "this should not appear" + mock_remove_ctn.side_effect = DockerAPIError( + message="my custom docker api error", + response=HttpResponse(content=fake_response_content)) + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], "localhost") + self.assertIn(InvalidPingResponse.__name__, res["traceback"]) + self.assertIn(invalid_ping_resp_msg, res["traceback"]) + + # No need to bother the students with this nonsense. + self.assertNotIn(DockerAPIError.__name__, res["traceback"]) + self.assertNotIn(fake_response_content, res["traceback"]) + + @skipIf(six.PY2, "PY2 doesn't support subTest") + def test_docker_container_ping_return_not_ok(self): + with ( + mock.patch("docker.client.Client.create_container")) as mock_create_ctn, ( # noqa + mock.patch("docker.client.Client.start")) as mock_ctn_start, ( + mock.patch("docker.client.Client.logs")) as mock_ctn_logs, ( + mock.patch("docker.client.Client.remove_container")) as mock_remove_ctn, ( # noqa + mock.patch("docker.client.Client.inspect_container")) as mock_inpect_ctn, ( # noqa + mock.patch("six.moves.http_client.HTTPConnection.request")) as mock_ctn_request, ( # noqa + mock.patch("six.moves.http_client.HTTPConnection.getresponse")) as mock_ctn_get_response: # noqa + + mock_create_ctn.return_value = {"Id": "someid"} + mock_ctn_start.side_effect = lambda x: None + mock_ctn_logs.side_effect = lambda x: None + mock_remove_ctn.return_value = None + fake_host_ip = "192.168.1.100" + fake_host_port = "69999" + + mock_inpect_ctn.return_value = { + "NetworkSettings": { + "Ports": {"%d/tcp" % RUNPY_PORT: ( + {"HostIp": fake_host_ip, "HostPort": fake_host_port}, + )} + }} + + # force timeout + with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): + with self.subTest( + case="Docker ping response not OK"): + mock_ctn_request.side_effect = lambda x, y: None + mock_ctn_get_response.return_value = six.BytesIO(b"NOT OK") + + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "uncaught_error") + self.assertEqual(res['message'], + "Timeout waiting for container.") + self.assertEqual(res["exec_host"], fake_host_ip) + self.assertIn(InvalidPingResponse.__name__, res["traceback"]) + + @skipIf(six.PY2, "PY2 doesn't support subTest") + def test_docker_container_runpy_timeout(self): + with ( + mock.patch("docker.client.Client.create_container")) as mock_create_ctn, ( # noqa + mock.patch("docker.client.Client.start")) as mock_ctn_start, ( + mock.patch("docker.client.Client.logs")) as mock_ctn_logs, ( + mock.patch("docker.client.Client.remove_container")) as mock_remove_ctn, ( # noqa + mock.patch("docker.client.Client.inspect_container")) as mock_inpect_ctn, ( # noqa + mock.patch("six.moves.http_client.HTTPConnection.request")) as mock_ctn_request, ( # noqa + mock.patch("six.moves.http_client.HTTPConnection.getresponse")) as mock_ctn_get_response: # noqa + + mock_create_ctn.return_value = {"Id": "someid"} + mock_ctn_start.side_effect = lambda x: None + mock_ctn_logs.side_effect = lambda x: None + mock_remove_ctn.return_value = None + fake_host_ip = "192.168.1.100" + fake_host_port = "69999" + + mock_inpect_ctn.return_value = { + "NetworkSettings": { + "Ports": {"%d/tcp" % RUNOC_PORT: ( + {"HostIp": fake_host_ip, "HostPort": fake_host_port}, + )} + }} + + with self.subTest( + case="Docker ping passed by runpy timed out"): + + # first request is ping, second request raise socket.timeout + mock_ctn_request.side_effect = [None, sock_timeout] + mock_ctn_get_response.return_value = six.BytesIO(b"OK") + + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=0) + self.assertEqual(res["result"], "timeout") + self.assertEqual(res["exec_host"], fake_host_ip) + + @skipIf(six.PY2, "PY2 doesn't support subTest") + def test_docker_container_runoc_retries_count(self): + with ( + mock.patch("course.page.code.request_octave_run")) as mock_req_run, ( # noqa + mock.patch("course.page.code.is_nuisance_failure")) as mock_is_nuisance_failure: # noqa + expected_result = "this is my custom result" + mock_req_run.return_value = {"result": expected_result} + with self.subTest(actual_retry_count=4): + mock_is_nuisance_failure.side_effect = [True, True, True, False] + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=5) + self.assertEqual(res["result"], expected_result) + self.assertEqual(mock_req_run.call_count, 4) + self.assertEqual(mock_is_nuisance_failure.call_count, 4) + + mock_req_run.reset_mock() + mock_is_nuisance_failure.reset_mock() + with self.subTest(actual_retry_count=2): + mock_is_nuisance_failure.side_effect = [True, True, True, False] + res = request_python_run_with_retries( + run_req={}, run_timeout=0.1, retry_count=1) + self.assertEqual(res["result"], expected_result) + self.assertEqual(mock_req_run.call_count, 2) + self.assertEqual(mock_is_nuisance_failure.call_count, 1) + + class IsNuisanceFailureTest(unittest.TestCase): # Testing is_nuisance_failure From 0e8768dca313939e592bc694afbd705d1e383dd3 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Sun, 30 Dec 2018 16:06:10 -0600 Subject: [PATCH 03/14] Flatten Docker image back into one. --- docker-image-run-octave/runoc-update/Dockerfile | 14 -------------- .../runoc-update/docker-build.sh | 4 ---- 2 files changed, 18 deletions(-) delete mode 100644 docker-image-run-octave/runoc-update/Dockerfile delete mode 100644 docker-image-run-octave/runoc-update/docker-build.sh diff --git a/docker-image-run-octave/runoc-update/Dockerfile b/docker-image-run-octave/runoc-update/Dockerfile deleted file mode 100644 index 634a3a567..000000000 --- a/docker-image-run-octave/runoc-update/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM inducer/runoc -MAINTAINER Neal Davis -EXPOSE 9942 -#RUN useradd runoc - -#RUN apt-get update && \ -# pip install oct2py - -RUN mkdir -p /opt/runoc -#ADD runoc /opt/runoc/ -COPY code_feedback.py /opt/runoc/ -COPY code_runoc_backend.py /opt/runoc/ - -RUN rm -Rf /root/.cache diff --git a/docker-image-run-octave/runoc-update/docker-build.sh b/docker-image-run-octave/runoc-update/docker-build.sh deleted file mode 100644 index fb4e6ed9b..000000000 --- a/docker-image-run-octave/runoc-update/docker-build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -cp ../../course/page/code_feedback.py ../../course/page/code_runoc_backend.py . -docker build --no-cache . -t inducer/runoc-update -rm code_feedback.py code_runoc_backend.py From 3491cd3818a171c4da66f3d5225ecd56134430b9 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Sun, 30 Dec 2018 16:11:15 -0600 Subject: [PATCH 04/14] Include Docker image by name. It needs to be built first. --- local_settings_example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/local_settings_example.py b/local_settings_example.py index 17d982f14..75ddeeb39 100644 --- a/local_settings_example.py +++ b/local_settings_example.py @@ -343,6 +343,7 @@ # A string containing the image ID of the docker image to be used to run # student Python code. Docker should download the image on first run. RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64" +RELATE_DOCKER_RUNOC_IMAGE = "inducer/runoc" # RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64-tensorflow" # (bigger, but includes TensorFlow) From 6f8980c76f46173b21833c119425874fddf0e1a7 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 16:55:58 -0500 Subject: [PATCH 05/14] First stab at a CodeQuestion interface. UNTESTED but pushed for backup --- bin/relate | 7 +- course/page/code.py | 1100 +++++------------ course/page/code_run_backend.py | 313 +++++ docker-image-run-octave/Dockerfile | 11 +- docker-image-run-octave/docker-build.sh | 5 +- docker-image-run-octave/{runoc => runcode} | 6 +- .../runoc-update/Dockerfile | 14 + .../runoc-update/docker-build.sh | 5 + docker-image-run-py/Dockerfile | 10 +- docker-image-run-py/docker-build.sh | 6 +- docker-image-run-py/{runpy => runcode} | 6 +- setup.cfg | 2 +- tests/base_test_mixins.py | 8 +- 13 files changed, 645 insertions(+), 848 deletions(-) create mode 100644 course/page/code_run_backend.py rename docker-image-run-octave/{runoc => runcode} (96%) create mode 100644 docker-image-run-octave/runoc-update/Dockerfile create mode 100644 docker-image-run-octave/runoc-update/docker-build.sh rename docker-image-run-py/{runpy => runcode} (96%) diff --git a/bin/relate b/bin/relate index 90879fd71..a5243fd86 100755 --- a/bin/relate +++ b/bin/relate @@ -52,7 +52,8 @@ def expand_yaml(yml_file, repo_root): def test_code_question(page_desc, repo_root): if page_desc.type not in [ "PythonCodeQuestion", - "PythonCodeQuestionWithHumanTextFeedback"]: + "PythonCodeQuestionWithHumanTextFeedback", + "OctaveCodeQuestion"]: return print(75*"-") @@ -64,11 +65,11 @@ def test_code_question(page_desc, repo_root): correct_code = getattr(page_desc, "correct_code", "") - from course.page.code_runpy_backend import \ + from course.page.code_run_backend import \ substitute_correct_code_into_test_code test_code = substitute_correct_code_into_test_code(test_code, correct_code) - from course.page.code_runpy_backend import run_code, package_exception + from course.page.code_run_backend import run_code, package_exception data_files = {} diff --git a/course/page/code.py b/course/page/code.py index d95ff2faa..017451e70 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -44,24 +44,23 @@ # DEBUGGING SWITCH: # True for 'spawn containers' (normal operation) -# False for 'just connect to localhost:RUNPY_PORT' for runpy' -SPAWN_CONTAINERS_FOR_RUNPY = True -SPAWN_CONTAINERS_FOR_RUNOC = True +# False for 'just connect to localhost:CODE_QUESTION_CONTAINER_PORT' for runcode' +SPAWN_CONTAINERS = True +# {{{ base code question -# {{{ python code question - -class PythonCodeForm(StyledForm): +class CodeForm(StyledForm): # prevents form submission with codemirror's empty textarea use_required_attribute = False def __init__(self, read_only, interaction_mode, initial_code, - data=None, *args, **kwargs): - super(PythonCodeForm, self).__init__(data, *args, **kwargs) + language_mode, data=None, *args, **kwargs): + super(CodeForm, self).__init__(data, *args, **kwargs) + self.language_mode = language_mode from course.utils import get_codemirror_widget cm_widget, cm_help_text = get_codemirror_widget( - language_mode="python", + language_mode=language_mode, interaction_mode=interaction_mode, read_only=read_only, @@ -84,7 +83,7 @@ def clean(self): pass -RUNPY_PORT = 9941 +CODE_QUESTION_CONTAINER_PORT = 9941 DOCKER_TIMEOUT = 15 @@ -92,7 +91,7 @@ class InvalidPingResponse(RuntimeError): pass -def request_python_run(run_req, run_timeout, image=None): +def request_run(run_req, run_timeout, image=None): import json from six.moves import http_client import docker @@ -100,7 +99,7 @@ def request_python_run(run_req, run_timeout, image=None): import errno from docker.errors import APIError as DockerAPIError - debug = False + debug = True if debug: def debug_print(s): print(s) @@ -108,7 +107,11 @@ def debug_print(s): def debug_print(s): pass - if SPAWN_CONTAINERS_FOR_RUNPY: + image = self.container_image + command_path = '/opt/runcode/runcode' + user = 'runcode' + + if SPAWN_CONTAINERS: docker_url = getattr(settings, "RELATE_DOCKER_URL", "unix://var/run/docker.sock") docker_tls = getattr(settings, "RELATE_DOCKER_TLS_CONFIG", @@ -119,13 +122,10 @@ def debug_print(s): timeout=DOCKER_TIMEOUT, version="1.19") - if image is None: - image = settings.RELATE_DOCKER_RUNPY_IMAGE - dresult = docker_cnx.create_container( image=image, command=[ - "/opt/runpy/runpy", + command_path, "-1"], host_config={ "Memory": 384*10**6, @@ -134,7 +134,7 @@ def debug_print(s): # Do not enable: matplotlib stops working if enabled. # "ReadonlyRootfs": True, }, - user="runpy") + user=user) container_id = dresult["Id"] else: @@ -150,7 +150,7 @@ def debug_print(s): container_props = docker_cnx.inspect_container(container_id) (port_info,) = (container_props - ["NetworkSettings"]["Ports"]["%d/tcp" % RUNPY_PORT]) + ["NetworkSettings"]["Ports"]["%d/tcp" % RUNOC_PORT]) port_host_ip = port_info.get("HostIp") if port_host_ip != "0.0.0.0": @@ -158,7 +158,7 @@ def debug_print(s): port = int(port_info["HostPort"]) else: - port = RUNPY_PORT + port = CODE_QUESTION_CONTAINER_PORT from time import time, sleep start_time = time() @@ -224,7 +224,7 @@ def check_timeout(): start_time = time() debug_print("BEFPOST") - connection.request('POST', '/run-python', json_run_req, headers) + connection.request('POST', '/run-%s'%language_mode, json_run_req, headers) debug_print("AFTPOST") http_response = connection.getresponse() @@ -261,6 +261,7 @@ def check_timeout(): # Oh well. No need to bother the students with this nonsense. pass + def is_nuisance_failure(result): if result["result"] != "uncaught_error": return False @@ -292,9 +293,9 @@ def is_nuisance_failure(result): return False -def request_python_run_with_retries(run_req, run_timeout, image=None, retry_count=3): +def request_run_with_retries(run_req, run_timeout, image=None, retry_count=3): while True: - result = request_python_run(run_req, run_timeout, image=image) + result = request_run(run_req, run_timeout, image=image) if retry_count and is_nuisance_failure(result): retry_count -= 1 @@ -303,11 +304,12 @@ def request_python_run_with_retries(run_req, run_timeout, image=None, retry_coun return result -class PythonCodeQuestion(PageBaseWithTitle, PageBaseWithValue): +class CodeQuestion(PageBaseWithTitle, PageBaseWithValue): """ - An auto-graded question allowing an answer consisting of Python code. + An auto-graded question allowing an answer consisting of code. All user code as well as all code specified as part of the problem - is in Python 3. + is in the specified language. This class should be treated as an + interface and used only as a superclass. If you are not including the :attr:`course.constants.flow_permission.change_answer` @@ -329,7 +331,7 @@ class PythonCodeQuestion(PageBaseWithTitle, PageBaseWithValue): .. attribute:: type - ``PythonCodeQuestion`` + ``CodeQuestion`` .. attribute:: is_optional_page @@ -360,7 +362,7 @@ class PythonCodeQuestion(PageBaseWithTitle, PageBaseWithValue): .. attribute:: setup_code Optional. - Python code to prepare the environment for the participants + Language-specific code to prepare the environment for the participants answer. .. attribute:: show_setup_code @@ -479,8 +481,9 @@ class PythonCodeQuestion(PageBaseWithTitle, PageBaseWithValue): * ``user_code``: The user code being tested, as a string. """ - def __init__(self, vctx, location, page_desc): - super(PythonCodeQuestion, self).__init__(vctx, location, page_desc) + def __init__(self, vctx, location, page_desc, language_mode): + super(CodeQuestion, self).__init__(vctx, location, page_desc) + self.language_mode = language_mode if vctx is not None and hasattr(page_desc, "data_files"): for data_file in page_desc.data_files: @@ -516,13 +519,13 @@ def __init__(self, vctx, location, page_desc): "access_rules/add_permssions/see_correctness.")) def required_attrs(self): - return super(PythonCodeQuestion, self).required_attrs() + ( + return super(CodeQuestion, self).required_attrs() + ( ("prompt", "markup"), ("timeout", (int, float)), ) def allowed_attrs(self): - return super(PythonCodeQuestion, self).allowed_attrs() + ( + return super(CodeQuestion, self).allowed_attrs() + ( ("setup_code", str), ("show_setup_code", bool), ("names_for_user", list), @@ -567,14 +570,14 @@ def make_form(self, page_context, page_data, if answer_data is not None: answer = {"answer": answer_data["answer"]} - form = PythonCodeForm( + form = CodeForm( not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), answer) else: answer = None - form = PythonCodeForm( + form = CodeForm( not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), @@ -584,7 +587,7 @@ def make_form(self, page_context, page_data, def process_form_post( self, page_context, page_data, post_data, files_data, page_behavior): - return PythonCodeForm( + return CodeForm( not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), @@ -602,7 +605,7 @@ def get_test_code(self): if correct_code is None: correct_code = "" - from .code_runpy_backend import substitute_correct_code_into_test_code + from .code_run_backend import substitute_correct_code_into_test_code return substitute_correct_code_into_test_code(test_code, correct_code) def grade(self, page_context, page_data, answer_data, grade_data): @@ -640,8 +643,9 @@ def transfer_attr(name): page_context.commit_sha).data).decode() try: - response_dict = request_python_run_with_retries(run_req, - run_timeout=self.page_desc.timeout) + response_dict = request_run_with_retries(run_req, + run_timeout=self.page_desc.timeout, + language_mode=self.language_mode) except Exception: from traceback import format_exc response_dict = { @@ -822,7 +826,7 @@ def normalize_code(s): correctness = 0 else: - raise RuntimeError("invalid runpy result: %s" % response.result) + raise RuntimeError("invalid runoc result: %s" % response.result) if hasattr(response, "feedback") and response.feedback: def sanitize(s): @@ -980,13 +984,212 @@ def normalized_answer(self, page_context, page_data, answer_data): return "
    %s
    " % escape(normalized_answer) def normalized_bytes_answer(self, page_context, page_data, answer_data): - if answer_data is None: - return None + if answer_data is None: + return None - return (".py", answer_data["answer"].encode("utf-8")) + suffix = self.suffix + return (suffix, answer_data["answer"].encode("utf-8")) # }}} +# {{{ python code question + +class PythonCodeForm(CodeForm): + def __init__(self, vctx, location, page_desc): + super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc, language_mode='python') + +class PythonCodeQuestion(CodeQuestion): + """ + An auto-graded question allowing an answer consisting of Python code. + All user code as well as all code specified as part of the problem + is in Python 3. + + If you are not including the + :attr:`course.constants.flow_permission.change_answer` + permission for your entire flow, you likely want to + include this snippet in your question definition: + + .. code-block:: yaml + + access_rules: + add_permissions: + - change_answer + + This will allow participants multiple attempts at getting + the right answer. + + .. attribute:: id + + |id-page-attr| + + .. attribute:: type + + ``PythonCodeQuestion`` + + .. attribute:: is_optional_page + + |is-optional-page-attr| + + .. attribute:: access_rules + + |access-rules-page-attr| + + .. attribute:: title + + |title-page-attr| + + .. attribute:: value + + |value-page-attr| + + .. attribute:: prompt + + The page's prompt, written in :ref:`markup`. + + .. attribute:: timeout + + A number, giving the number of seconds for which setup code, + the given answer code, and the test code (combined) will be + allowed to run. + + .. attribute:: setup_code + + Optional. + Python code to prepare the environment for the participants + answer. + + .. attribute:: show_setup_code + + Optional. ``True`` or ``False``. If true, the :attr:`setup_code` + will be shown to the participant. + + .. attribute:: names_for_user + + Optional. + Symbols defined at the end of the :attr:`setup_code` that will be + made available to the participant's code. + + A deep copy (using the standard library function :func:`copy.deepcopy`) + of these values is made, to prevent the user from modifying trusted + state of the grading code. + + .. attribute:: names_from_user + + Optional. + Symbols that the participant's code is expected to define. + These will be made available to the :attr:`test_code`. + + .. attribute:: test_code + + Optional. + Code that will be run to determine the correctness of a + student-provided solution. Will have access to variables in + :attr:`names_from_user` (which will be *None*) if not provided. Should + never raise an exception. + + This may contain the marker "###CORRECT_CODE###", which will + be replaced with the contents of :attr:`correct_code`, with + each line indented to the same depth as where the marker + is found. The line with this marker is only allowed to have + white space and the marker on it. + + .. attribute:: show_test_code + + Optional. ``True`` or ``False``. If true, the :attr:`test_code` + will be shown to the participant. + + .. attribute:: correct_code_explanation + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). This is shown before + :attr:`correct_code` as an explanation. + + .. attribute:: correct_code + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). + + .. attribute:: initial_code + + Optional. + Code present in the code input field when the participant first starts + working on their solution. + + .. attribute:: data_files + + Optional. + A list of file names in the :ref:`git-repo` whose contents will be made + available to :attr:`setup_code` and :attr:`test_code` through the + ``data_files`` dictionary. (see below) + + .. attribute:: single_submission + + Optional, a Boolean. If the question does not allow multiple submissions + based on its :attr:`access_rules` (not the ones of the flow), a warning + is shown. Setting this attribute to True will silence the warning. + + The following symbols are available in :attr:`setup_code` and :attr:`test_code`: + + * ``GradingComplete``: An exception class that can be raised to indicated + that the grading code has concluded. + + * ``feedback``: A class instance with the following interface:: + + feedback.set_points(0.5) # 0<=points<=1 (usually) + feedback.add_feedback("This was wrong") + + # combines the above two and raises GradingComplete + feedback.finish(0, "This was wrong") + + feedback.check_numpy_array_sanity(name, num_axes, data) + + feedback.check_numpy_array_features(name, ref, data, report_failure=True) + + feedback.check_numpy_array_allclose(name, ref, data, + accuracy_critical=True, rtol=1e-5, atol=1e-8, + report_success=True, report_failure=True) + # If report_failure is True, this function will only return + # if *data* passes the tests. It will return *True* in this + # case. + # + # If report_failure is False, this function will always return, + # and the return value will indicate whether *data* passed the + # accuracy/shape/kind checks. + + feedback.check_list(name, ref, data, entry_type=None) + + feedback.check_scalar(name, ref, data, accuracy_critical=True, + rtol=1e-5, atol=1e-8, report_success=True, report_failure=True) + # returns True if accurate + + feedback.call_user(f, *args, **kwargs) + # Calls a user-supplied function and prints an appropriate + # feedback message in case of failure. + + * ``data_files``: A dictionary mapping file names from :attr:`data_files` + to :class:`bytes` instances with that file's contents. + + * ``user_code``: The user code being tested, as a string. + """ + + @property + def language_mode(self): + return 'python' + + @property + def container_image(self): + return settings.RELATE_DOCKER_RUNPY_IMAGE + + @property + def suffix(self): + return '.py' + + def __init__(self, vctx, location, page_desc, language_mode='python'): + super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc) + +# }}} # {{{ python code question with human feedback @@ -1192,292 +1395,43 @@ def grade(self, page_context, page_data, answer_data, grade_data): # {{{ octave code question -class OctaveCodeForm(StyledForm): - # prevents form submission with codemirror's empty textarea - use_required_attribute = False +class OctaveCodeForm(CodeForm): + def __init__(self, vctx, location, page_desc): + super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc, language_mode='octave') - def __init__(self, read_only, interaction_mode, initial_code, - data=None, *args, **kwargs): - super(OctaveCodeForm, self).__init__(data, *args, **kwargs) #TODO OctaveCodeForm +class OctaveCodeQuestion(CodeQuestion): + """ + An auto-graded question allowing an answer consisting of Octave code. + All user code as well as all code specified as part of the problem + is in Octave 4.2+. - from course.utils import get_codemirror_widget - cm_widget, cm_help_text = get_codemirror_widget( - language_mode="octave", - interaction_mode=interaction_mode, - read_only=read_only, + If you are not including the + :attr:`course.constants.flow_permission.change_answer` + permission for your entire flow, you likely want to + include this snippet in your question definition: - # Automatically focus the text field once there has - # been some input. - autofocus=( - not read_only - and (data is not None and "answer" in data))) + .. code-block:: yaml - self.fields["answer"] = forms.CharField(required=True, - initial=initial_code, - help_text=cm_help_text, - widget=cm_widget, - label=_("Answer")) + access_rules: + add_permissions: + - change_answer - self.style_codemirror_widget() + This will allow participants multiple attempts at getting + the right answer. - def clean(self): - # FIXME Should try compilation - pass + .. attribute:: id + |id-page-attr| -RUNOC_PORT = 9942 -DOCKER_TIMEOUT = 15 + .. attribute:: type + ``OctaveCodeQuestion`` -class InvalidPingResponse(RuntimeError): - pass + .. attribute:: is_optional_page + |is-optional-page-attr| -def request_octave_run(run_req, run_timeout, image=None): - import json - from six.moves import http_client - import docker - import socket - import errno - from docker.errors import APIError as DockerAPIError - - debug = True - if debug: - def debug_print(s): - print(s) - else: - def debug_print(s): - pass - - if SPAWN_CONTAINERS_FOR_RUNOC: - docker_url = getattr(settings, "RELATE_DOCKER_URL", - "unix://var/run/docker.sock") - docker_tls = getattr(settings, "RELATE_DOCKER_TLS_CONFIG", - None) - docker_cnx = docker.Client( - base_url=docker_url, - tls=docker_tls, - timeout=DOCKER_TIMEOUT, - version="1.19") - - if image is None: - image = settings.RELATE_DOCKER_RUNOC_IMAGE - - dresult = docker_cnx.create_container( - image=image, - command=[ - "/opt/runoc/runoc", - "-1"], - host_config={ - "Memory": 384*10**6, - "MemorySwap": -1, - "PublishAllPorts": True, - # Do not enable: matplotlib stops working if enabled. - # "ReadonlyRootfs": True, - }, - user="runoc") - - container_id = dresult["Id"] - else: - container_id = None - - connect_host_ip = 'localhost' - - try: - # FIXME: Prohibit networking - - if container_id is not None: - docker_cnx.start(container_id) - - container_props = docker_cnx.inspect_container(container_id) - (port_info,) = (container_props - ["NetworkSettings"]["Ports"]["%d/tcp" % RUNOC_PORT]) - port_host_ip = port_info.get("HostIp") - - if port_host_ip != "0.0.0.0": - connect_host_ip = port_host_ip - - port = int(port_info["HostPort"]) - else: - port = RUNOC_PORT - - from time import time, sleep - start_time = time() - - # {{{ ping until response received - - from traceback import format_exc - - def check_timeout(): - if time() - start_time < DOCKER_TIMEOUT: - sleep(0.1) - # and retry - else: - return { - "result": "uncaught_error", - "message": "Timeout waiting for container.", - "traceback": "".join(format_exc()), - "exec_host": connect_host_ip, - } - - while True: - try: - connection = http_client.HTTPConnection(connect_host_ip, port) - - connection.request('GET', '/ping') - - response = connection.getresponse() - response_data = response.read().decode() - - if response_data != "OK": - raise InvalidPingResponse() - - break - - except (http_client.BadStatusLine, InvalidPingResponse): - ct_res = check_timeout() - if ct_res is not None: - return ct_res - - except socket.error as e: - if e.errno in [errno.ECONNRESET, errno.ECONNREFUSED]: - ct_res = check_timeout() - if ct_res is not None: - return ct_res - - else: - raise - - # }}} - - debug_print("PING SUCCESSFUL") - - try: - # Add a second to accommodate 'wire' delays - connection = http_client.HTTPConnection(connect_host_ip, port, - timeout=1 + run_timeout) - - headers = {'Content-type': 'application/json'} - - json_run_req = json.dumps(run_req).encode("utf-8") - - from time import time - start_time = time() - - debug_print("BEFPOST") - connection.request('POST', '/run-octave', json_run_req, headers) - debug_print("AFTPOST") - - http_response = connection.getresponse() - debug_print("GETR") - response_data = http_response.read().decode("utf-8") - debug_print("READR") - - end_time = time() - - result = json.loads(response_data) - - result["feedback"] = (result.get("feedback", []) - + ["Execution time: %.1f s -- Time limit: %.1f s" - % (end_time - start_time, run_timeout)]) - - result["exec_host"] = connect_host_ip - - return result - - except socket.timeout: - return { - "result": "timeout", - "exec_host": connect_host_ip, - } - finally: - if container_id is not None: - debug_print("-----------BEGIN DOCKER LOGS for %s" % container_id) - debug_print(docker_cnx.logs(container_id)) - debug_print("-----------END DOCKER LOGS for %s" % container_id) - - try: - docker_cnx.remove_container(container_id, force=True) - except DockerAPIError: - # Oh well. No need to bother the students with this nonsense. - pass - - -def is_nuisance_failure(result): - if result["result"] != "uncaught_error": - return False - - if "traceback" in result: - if "BadStatusLine" in result["traceback"]: - - # Occasionally, we fail to send a POST to the container, even after - # the inital ping GET succeeded, for (for now) mysterious reasons. - # Just try again. - - return True - - if "bind: address already in use" in result["traceback"]: - # https://github.com/docker/docker/issues/8714 - - return True - - if ("requests.packages.urllib3.exceptions.NewConnectionError" - in result["traceback"]): - return True - - if "http.client.RemoteDisconnected" in result["traceback"]: - return True - - if "[Errno 113] No route to host" in result["traceback"]: - return True - - return False - - -def request_octave_run_with_retries(run_req, run_timeout, image=None, retry_count=3): - while True: - result = request_octave_run(run_req, run_timeout, image=image) - - if retry_count and is_nuisance_failure(result): - retry_count -= 1 - continue - - return result - - -class OctaveCodeQuestion(PageBaseWithTitle, PageBaseWithValue): - """ - An auto-graded question allowing an answer consisting of Octave code. - All user code as well as all code specified as part of the problem - is in Octave 4.2+. - - If you are not including the - :attr:`course.constants.flow_permission.change_answer` - permission for your entire flow, you likely want to - include this snippet in your question definition: - - .. code-block:: yaml - - access_rules: - add_permissions: - - change_answer - - This will allow participants multiple attempts at getting - the right answer. - - .. attribute:: id - - |id-page-attr| - - .. attribute:: type - - ``OctaveCodeQuestion`` - - .. attribute:: is_optional_page - - |is-optional-page-attr| - - .. attribute:: access_rules + .. attribute:: access_rules |access-rules-page-attr| @@ -1622,512 +1576,20 @@ class OctaveCodeQuestion(PageBaseWithTitle, PageBaseWithValue): * ``user_code``: The user code being tested, as a string. """ - def __init__(self, vctx, location, page_desc): - super(OctaveCodeQuestion, self).__init__(vctx, location, page_desc) - - if vctx is not None and hasattr(page_desc, "data_files"): - for data_file in page_desc.data_files: - try: - if not isinstance(data_file, str): - raise ObjectDoesNotExist() - - from course.content import get_repo_blob - get_repo_blob(vctx.repo, data_file, vctx.commit_sha) - except ObjectDoesNotExist: - raise ValidationError( - string_concat( - "%(location)s: ", - _("data file '%(file)s' not found")) - % {"location": location, "file": data_file}) - - if not getattr(page_desc, "single_submission", False) and vctx is not None: - is_multi_submit = False - - if hasattr(page_desc, "access_rules"): - if hasattr(page_desc.access_rules, "add_permissions"): - if (flow_permission.change_answer - in page_desc.access_rules.add_permissions): - is_multi_submit = True + @property + def language_mode(self): + return 'octave' - if not is_multi_submit: - vctx.add_warning(location, _("code question does not explicitly " - "allow multiple submission. Either add " - "access_rules/add_permssions/change_answer " - "or add 'single_submission: True' to confirm that you intend " - "for only a single submission to be allowed. " - "While you're at it, consider adding " - "access_rules/add_permssions/see_correctness.")) + @property + def container_image(self): + return settings.RELATE_DOCKER_RUNOC_IMAGE - def required_attrs(self): - return super(OctaveCodeQuestion, self).required_attrs() + ( - ("prompt", "markup"), - ("timeout", (int, float)), - ) - - def allowed_attrs(self): - return super(OctaveCodeQuestion, self).allowed_attrs() + ( - ("setup_code", str), - ("show_setup_code", bool), - ("names_for_user", list), - ("names_from_user", list), - ("test_code", str), - ("show_test_code", bool), - ("correct_code_explanation", "markup"), - ("correct_code", str), - ("initial_code", str), - ("data_files", list), - ("single_submission", bool), - ) - - def _initial_code(self): - result = getattr(self.page_desc, "initial_code", None) - if result is not None: - return result.strip() - else: - return result - - def markup_body_for_title(self): - return self.page_desc.prompt - - def body(self, page_context, page_data): - from django.template.loader import render_to_string - return render_to_string( - "course/prompt-code-question.html", - { - "prompt_html": - markup_to_html(page_context, self.page_desc.prompt), - "initial_code": self._initial_code(), - "show_setup_code": getattr( - self.page_desc, "show_setup_code", False), - "setup_code": getattr(self.page_desc, "setup_code", ""), - "show_test_code": getattr( - self.page_desc, "show_test_code", False), - "test_code": getattr(self.page_desc, "test_code", ""), - }) - - def make_form(self, page_context, page_data, - answer_data, page_behavior): - - if answer_data is not None: - answer = {"answer": answer_data["answer"]} - form = OctaveCodeForm( - not page_behavior.may_change_answer, - get_editor_interaction_mode(page_context), - self._initial_code(), - answer) - else: - answer = None - form = OctaveCodeForm( - not page_behavior.may_change_answer, - get_editor_interaction_mode(page_context), - self._initial_code(), - ) - - return form - - def process_form_post( - self, page_context, page_data, post_data, files_data, page_behavior): - return OctaveCodeForm( - not page_behavior.may_change_answer, - get_editor_interaction_mode(page_context), - self._initial_code(), - post_data, files_data) - - def answer_data(self, page_context, page_data, form, files_data): - return {"answer": form.cleaned_data["answer"].strip()} - - def get_test_code(self): - test_code = getattr(self.page_desc, "test_code", None) - if test_code is None: - return test_code - - correct_code = getattr(self.page_desc, "correct_code", None) - if correct_code is None: - correct_code = "" - - from .code_runoc_backend import substitute_correct_code_into_test_code - return substitute_correct_code_into_test_code(test_code, correct_code) - - def grade(self, page_context, page_data, answer_data, grade_data): - if answer_data is None: - return AnswerFeedback(correctness=0, - feedback=_("No answer provided.")) - - user_code = answer_data["answer"] - - # {{{ request run - - run_req = {"compile_only": False, "user_code": user_code} - - def transfer_attr(name): - if hasattr(self.page_desc, name): - run_req[name] = getattr(self.page_desc, name) - - transfer_attr("setup_code") - transfer_attr("names_for_user") - transfer_attr("names_from_user") - - run_req["test_code"] = self.get_test_code() - - if hasattr(self.page_desc, "data_files"): - run_req["data_files"] = {} - - from course.content import get_repo_blob - - # TODO: set this up to work with M-files for Octave functions - for data_file in self.page_desc.data_files: - from base64 import b64encode - run_req["data_files"][data_file] = \ - b64encode( - get_repo_blob( - page_context.repo, data_file, - page_context.commit_sha).data).decode() - - try: - response_dict = request_octave_run_with_retries(run_req, - run_timeout=self.page_desc.timeout) - except Exception: - from traceback import format_exc - response_dict = { - "result": "uncaught_error", - "message": "Error connecting to container", - "traceback": "".join(format_exc()), - } - - # }}} - - feedback_bits = [] - - correctness = None - - if "points" in response_dict: - correctness = response_dict["points"] - try: - feedback_bits.append( - "

    %s

    " - % _(get_auto_feedback(correctness))) - except Exception as e: - correctness = None - response_dict["result"] = "setup_error" - response_dict["message"] = ( - "%s: %s" % (type(e).__name__, str(e)) - ) - - # {{{ send email if the grading code broke - - if response_dict["result"] in [ - "uncaught_error", - "setup_compile_error", - "setup_error", - "test_compile_error", - "test_error"]: - error_msg_parts = ["RESULT: %s" % response_dict["result"]] - for key, val in sorted(response_dict.items()): - if (key not in ["result", "figures"] - and val - and isinstance(val, six.string_types)): - error_msg_parts.append("-------------------------------------") - error_msg_parts.append(key) - error_msg_parts.append("-------------------------------------") - error_msg_parts.append(val) - error_msg_parts.append("-------------------------------------") - error_msg_parts.append("user code") - error_msg_parts.append("-------------------------------------") - error_msg_parts.append(user_code) - error_msg_parts.append("-------------------------------------") - - error_msg = "\n".join(error_msg_parts) - - from relate.utils import local_now, format_datetime_local - from course.utils import LanguageOverride - with LanguageOverride(page_context.course): - from relate.utils import render_email_template - message = render_email_template( - "course/broken-code-question-email.txt", { - "site": getattr(settings, "RELATE_BASE_URL"), - "page_id": self.page_desc.id, - "course": page_context.course, - "error_message": error_msg, - "review_uri": page_context.page_uri, - "time": format_datetime_local(local_now()) - }) - - if ( - not page_context.in_sandbox - and not is_nuisance_failure(response_dict)): - try: - from django.core.mail import EmailMessage - msg = EmailMessage("".join(["[%s:%s] ", - _("code question execution failed")]) - % ( - page_context.course.identifier, - page_context.flow_session.flow_id - if page_context.flow_session is not None - else _("")), - message, - settings.ROBOT_EMAIL_FROM, - [page_context.course.notify_email]) - - from relate.utils import get_outbound_mail_connection - msg.connection = get_outbound_mail_connection("robot") - msg.send() - - except Exception: - from traceback import format_exc - feedback_bits.append( - six.text_type(string_concat( - "

    ", - _( - "Both the grading code and the attempt to " - "notify course staff about the issue failed. " - "Please contact the course or site staff and " - "inform them of this issue, mentioning this " - "entire error message:"), - "

    ", - "

    ", - _( - "Sending an email to the course staff about the " - "following failure failed with " - "the following error message:"), - "

    ",
-                                "".join(format_exc()),
-                                "
    ", - _("The original failure message follows:"), - "

    "))) - - # }}} - - if hasattr(self.page_desc, "correct_code"): - def normalize_code(s): - return (s - .replace(" ", "") - .replace("\r", "") - .replace("\n", "") - .replace("\t", "")) - - if (normalize_code(user_code) - == normalize_code(self.page_desc.correct_code)): - feedback_bits.append( - "

    %s

    " - % _("It looks like you submitted code that is identical to " - "the reference solution. This is not allowed.")) - - from relate.utils import dict_to_struct - response = dict_to_struct(response_dict) - - bulk_feedback_bits = [] - - if response.result == "success": - pass - elif response.result in [ - "uncaught_error", - "setup_compile_error", - "setup_error", - "test_compile_error", - "test_error"]: - feedback_bits.append("".join([ - "

    ", - _( - "The grading code failed. Sorry about that. " - "The staff has been informed, and if this problem is " - "due to an issue with the grading code, " - "it will be fixed as soon as possible. " - "In the meantime, you'll see a traceback " - "below that may help you figure out what went wrong." - ), - "

    "])) - elif response.result == "timeout": - feedback_bits.append("".join([ - "

    ", - _( - "Your code took too long to execute. The problem " - "specifies that your code may take at most %s seconds " - "to run. " - "It took longer than that and was aborted." - ), - "

    "]) - % self.page_desc.timeout) - - correctness = 0 - elif response.result == "user_compile_error": - feedback_bits.append("".join([ - "

    ", - _("Your code failed to compile. An error message is " - "below."), - "

    "])) - - correctness = 0 - elif response.result == "user_error": - feedback_bits.append("".join([ - "

    ", - _("Your code failed with an exception. " - "A traceback is below."), - "

    "])) - - correctness = 0 - else: - raise RuntimeError("invalid runoc result: %s" % response.result) - - if hasattr(response, "feedback") and response.feedback: - def sanitize(s): - import bleach - return bleach.clean(s, tags=["p", "pre"]) - feedback_bits.append("".join([ - "

    ", - _("Here is some feedback on your code"), - ":" - "

      %s

    "]) % - "".join( - "
  • %s
    • " % sanitize(fb_item) - for fb_item in response.feedback)) - if hasattr(response, "traceback") and response.traceback: - feedback_bits.append("".join([ - "

    ", - _("This is the exception traceback"), - ":" - "

    %s

    "]) % escape(response.traceback)) - if hasattr(response, "exec_host") and response.exec_host != "localhost": - import socket - try: - exec_host_name, dummy, dummy = socket.gethostbyaddr( - response.exec_host) - except socket.error: - exec_host_name = response.exec_host - - feedback_bits.append("".join([ - "

    ", - _("Your code ran on %s.") % exec_host_name, - "

    "])) - - if hasattr(response, "stdout") and response.stdout: - bulk_feedback_bits.append("".join([ - "

    ", - _("Your code printed the following output"), - ":" - "

    %s

    "]) - % escape(response.stdout)) - if hasattr(response, "stderr") and response.stderr: - bulk_feedback_bits.append("".join([ - "

    ", - _("Your code printed the following error messages"), - ":" - "

    %s

    "]) % escape(response.stderr)) - if hasattr(response, "figures") and response.figures: - fig_lines = ["".join([ - "

    ", - _("Your code produced the following plots"), - ":

    "]), - '
    ', - ] - - for nr, mime_type, b64data in response.figures: - if mime_type in ["image/jpeg", "image/png"]: - fig_lines.extend([ - "".join([ - "
    ", - _("Figure"), "%d
    "]) % nr, - '
    Figure %d
    ' - % (nr, mime_type, b64data)]) - - fig_lines.append("
    ") - bulk_feedback_bits.extend(fig_lines) - - # {{{ html output / santization - - if hasattr(response, "html") and response.html: - def is_allowed_data_uri(allowed_mimetypes, uri): - import re - m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri) - if not m: - return False - - mimetype = m.group(1) - return mimetype in allowed_mimetypes - - def sanitize(s): - import bleach - - def filter_audio_attributes(tag, name, value): - if name in ["controls"]: - return True - else: - return False - - def filter_source_attributes(tag, name, value): - if name in ["type"]: - return True - elif name == "src": - if is_allowed_data_uri([ - "audio/wav", - ], value): - return bleach.sanitizer.VALUE_SAFE - else: - return False - else: - return False - - def filter_img_attributes(tag, name, value): - if name in ["alt", "title"]: - return True - elif name == "src": - return is_allowed_data_uri([ - "image/png", - "image/jpeg", - ], value) - else: - return False - - if not isinstance(s, six.text_type): - return _("(Non-string in 'HTML' output filtered out)") - - return bleach.clean(s, - tags=bleach.ALLOWED_TAGS + ["audio", "video", "source"], - attributes={ - "audio": filter_audio_attributes, - "source": filter_source_attributes, - "img": filter_img_attributes, - }) - - bulk_feedback_bits.extend( - sanitize(snippet) for snippet in response.html) - - # }}} - - return AnswerFeedback( - correctness=correctness, - feedback="\n".join(feedback_bits), - bulk_feedback="\n".join(bulk_feedback_bits)) - - def correct_answer(self, page_context, page_data, answer_data, grade_data): - result = "" - - if hasattr(self.page_desc, "correct_code_explanation"): - result += markup_to_html( - page_context, - self.page_desc.correct_code_explanation) - - if hasattr(self.page_desc, "correct_code"): - result += ("".join([ - _("The following code is a valid answer"), - ": 
    %s
    "]) - % escape(self.page_desc.correct_code)) - - return result - - def normalized_answer(self, page_context, page_data, answer_data): - if answer_data is None: - return None - - normalized_answer = answer_data["answer"] - - from django.utils.html import escape - return "
    %s
    " % escape(normalized_answer) - - def normalized_bytes_answer(self, page_context, page_data, answer_data): - if answer_data is None: - return None + @property + def suffix(self): + return '.m' - return (".m", answer_data["answer"].encode("utf-8")) + def __init__(self, vctx, location, page_desc, language_mode='octave'): + super(OctaveCodeQuestion, self).__init__(self, vctx, location, page_desc) # }}} diff --git a/course/page/code_run_backend.py b/course/page/code_run_backend.py new file mode 100644 index 000000000..b018db5cf --- /dev/null +++ b/course/page/code_run_backend.py @@ -0,0 +1,313 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import sys +import traceback + +try: + from .code_feedback import Feedback, GradingComplete +except SystemError: + from code_feedback import Feedback, GradingComplete # type: ignore +except ImportError: + from code_feedback import Feedback, GradingComplete # type: ignore + + +__doc__ = """ +PROTOCOL +======== + +.. class:: Request + + .. attribute:: setup_code + + .. attribute:: names_for_user + + .. attribute:: user_code + + .. attribute:: names_from_user + + .. attribute:: test_code + + .. attribute:: data_files + + A dictionary from data file names to their + base64-cencoded contents. + Optional. + + .. attribute:: compile_only + + :class:`bool` + +.. class Response:: + .. attribute:: result + + One of + + * ``success`` + * ``timeout`` + * ``uncaught_error`` + * ``setup_compile_error`` + * ``setup_error``, + * ``user_compile_error`` + * ``user_error`` + * ``test_compile_error`` + * ``test_error`` + + Always present. + + .. attribute:: message + + Optional. + + .. attribute:: traceback + + Optional. + + .. attribute:: stdout + + Whatever came out of stdout. + + Optional. + + .. attribute:: stderr + + Whatever came out of stderr. + + Optional. + + .. attribute:: figures + + A list of ``(index, mime_type, string)``, where *string* is a + base64-encoded representation of the figure. *index* will usually + correspond to the matplotlib figure number. + + Optional. + + .. attribute:: html + + A list of HTML strings generated. These are aggressively sanitized + before being rendered. + + .. attribute:: points + + A number between 0 and 1 (inclusive). + + Present on ``success`` if :attr:`Request.compile_only` is *False*. + + .. attribute:: feedback + + A list of strings. + + Present on ``success`` if :attr:`Request.compile_only` is *False*. +""" + + +# {{{ tools + +class Struct(object): + def __init__(self, entries): + for name, val in entries.items(): + self.__dict__[name] = val + + def __repr__(self): + return repr(self.__dict__) + +# }}} + + +def substitute_correct_code_into_test_code(test_code, correct_code): + import re + CORRECT_CODE_TAG = re.compile(r"^(\s*)###CORRECT_CODE###\s*$") # noqa + + new_test_code_lines = [] + for l in test_code.split("\n"): + match = CORRECT_CODE_TAG.match(l) + if match is not None: + prefix = match.group(1) + for cc_l in correct_code.split("\n"): + new_test_code_lines.append(prefix+cc_l) + else: + new_test_code_lines.append(l) + + return "\n".join(new_test_code_lines) + + +def package_exception(result, what): + tp, val, tb = sys.exc_info() + result["result"] = what + result["message"] = "%s: %s" % (tp.__name__, str(val)) + result["traceback"] = "".join( + traceback.format_exception(tp, val, tb)) + + +def run_code(result, run_req): + # {{{ silence matplotlib font cache warnings + + import warnings + warnings.filterwarnings( + "ignore", message="Matplotlib is building the font cache.*") + + # }}} + + # {{{ compile code + + if getattr(run_req, "setup_code", None): + try: + setup_code = compile( + run_req.setup_code, "[setup code]", 'exec') + except Exception: + package_exception(result, "setup_compile_error") + return + else: + setup_code = None + + try: + user_code = compile( + run_req.user_code, "[user code]", 'exec') + except Exception: + package_exception(result, "user_compile_error") + return + + if getattr(run_req, "test_code", None): + try: + test_code = compile( + run_req.test_code, "[test code]", 'exec') + except Exception: + package_exception(result, "test_compile_error") + return + else: + test_code = None + + # }}} + + if hasattr(run_req, "compile_only") and run_req.compile_only: + result["result"] = "success" + return + + # {{{ run code + + data_files = {} + if hasattr(run_req, "data_files"): + from base64 import b64decode + for name, contents in run_req.data_files.items(): + data_files[name] = b64decode(contents.encode()) + + generated_html = [] + result["html"] = generated_html + + def output_html(s): + generated_html.append(s) + + feedback = Feedback() + maint_ctx = { + "feedback": feedback, + "user_code": run_req.user_code, + "data_files": data_files, + "output_html": output_html, + "GradingComplete": GradingComplete, + } + + if setup_code is not None: + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.setup_code + exec(setup_code, maint_ctx) + except Exception: + package_exception(result, "setup_error") + return + + user_ctx = {} + if hasattr(run_req, "names_for_user"): + for name in run_req.names_for_user: + if name not in maint_ctx: + result["result"] = "setup_error" + result["message"] = "Setup code did not define '%s'." % name + + user_ctx[name] = maint_ctx[name] + + from copy import deepcopy + user_ctx = deepcopy(user_ctx) + + try: + user_ctx["_MODULE_SOURCE_CODE"] = run_req.user_code + exec(user_code, user_ctx) + except Exception: + package_exception(result, "user_error") + return + + # {{{ export plots + + if "matplotlib" in sys.modules: + import matplotlib.pyplot as pt + from io import BytesIO + from base64 import b64encode + + format = "png" + mime = "image/png" + figures = [] + + for fignum in pt.get_fignums(): + pt.figure(fignum) + bio = BytesIO() + try: + pt.savefig(bio, format=format) + except Exception: + pass + else: + figures.append( + (fignum, mime, b64encode(bio.getvalue()).decode())) + + result["figures"] = figures + + # }}} + + if hasattr(run_req, "names_from_user"): + for name in run_req.names_from_user: + if name not in user_ctx: + feedback.add_feedback( + "Required answer variable '%s' is not defined." + % name) + maint_ctx[name] = None + else: + maint_ctx[name] = user_ctx[name] + + if test_code is not None: + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.test_code + exec(test_code, maint_ctx) + except GradingComplete: + pass + except Exception: + package_exception(result, "test_error") + return + + result["points"] = feedback.points + result["feedback"] = feedback.feedback_items + + # }}} + + result["result"] = "success" + +# vim: foldmethod=marker diff --git a/docker-image-run-octave/Dockerfile b/docker-image-run-octave/Dockerfile index 6ce5f00c6..0c9ba5e12 100644 --- a/docker-image-run-octave/Dockerfile +++ b/docker-image-run-octave/Dockerfile @@ -1,7 +1,7 @@ FROM inducer/debian-amd64-minbase MAINTAINER Neal Davis EXPOSE 9942 -RUN useradd runoc +RUN useradd runcode # Based on `compdatasci/octave-desktop` Docker image @@ -49,10 +49,11 @@ RUN apt-get update && \ RUN apt-get clean RUN fc-cache -RUN mkdir -p /opt/runoc -ADD runoc /opt/runoc/ -COPY code_feedback.py /opt/runoc/ -COPY code_runoc_backend.py /opt/runoc/ +# TODO XXX perhaps I should genericize this as well +RUN mkdir -p /opt/runcode +ADD runcode /opt/runcode/ +COPY code_feedback.py /opt/runcode/ +COPY code_run_backend.py /opt/runcode/ # currently no graphics support #RUN sed -i s/TkAgg/Agg/ /etc/matplotlibrc diff --git a/docker-image-run-octave/docker-build.sh b/docker-image-run-octave/docker-build.sh index eb04820fa..dbcf6b1de 100755 --- a/docker-image-run-octave/docker-build.sh +++ b/docker-image-run-octave/docker-build.sh @@ -1,4 +1,5 @@ #! /bin/sh -cp ../course/page/code_feedback.py ../course/page/code_runoc_backend.py . +cp ../course/page/code_feedback.py . +cp ../course/page/code_runoc_backend.py code_run_backend.py docker build --no-cache . -t inducer/runoc -rm code_feedback.py code_runoc_backend.py +rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-octave/runoc b/docker-image-run-octave/runcode similarity index 96% rename from docker-image-run-octave/runoc rename to docker-image-run-octave/runcode index 3a5f6670f..bc756f5d2 100755 --- a/docker-image-run-octave/runoc +++ b/docker-image-run-octave/runcode @@ -30,18 +30,18 @@ import json import sys import io try: - from code_runoc_backend import Struct, run_code, package_exception + from code_run_backend import Struct, run_code, package_exception except ImportError: try: # When faking a container for unittest - from course.page.code_runoc_backend import ( + from course.page.code_run_backend import ( Struct, run_code, package_exception) except ImportError: # When debugging, i.e., run "python runpy" command line import os sys.path.insert(0, os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir))) - from course.page.code_runoc_backend import ( + from course.page.code_run_backend import ( Struct, run_code, package_exception) from http.server import BaseHTTPRequestHandler diff --git a/docker-image-run-octave/runoc-update/Dockerfile b/docker-image-run-octave/runoc-update/Dockerfile new file mode 100644 index 000000000..cf83a4e6a --- /dev/null +++ b/docker-image-run-octave/runoc-update/Dockerfile @@ -0,0 +1,14 @@ +FROM inducer/runoc +MAINTAINER Neal Davis +EXPOSE 9942 +#RUN useradd runcode + +#RUN apt-get update && \ +# pip install oct2py + +RUN mkdir -p /opt/runcode +#ADD runoc /opt/runcode/ +COPY code_feedback.py /opt/runcode/ +COPY code_run_backend.py /opt/runcode/ + +RUN rm -Rf /root/.cache diff --git a/docker-image-run-octave/runoc-update/docker-build.sh b/docker-image-run-octave/runoc-update/docker-build.sh new file mode 100644 index 000000000..364bad966 --- /dev/null +++ b/docker-image-run-octave/runoc-update/docker-build.sh @@ -0,0 +1,5 @@ +#! /bin/sh +cp ../../course/page/code_feedback.py . +cp ../../course/page/code_runoc_backend.py code_run_backend.py +docker build --no-cache . -t inducer/runoc-update +rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-py/Dockerfile b/docker-image-run-py/Dockerfile index bf98e050e..6f620d2ec 100644 --- a/docker-image-run-py/Dockerfile +++ b/docker-image-run-py/Dockerfile @@ -1,7 +1,7 @@ FROM inducer/debian-amd64-minbase MAINTAINER Andreas Kloeckner EXPOSE 9941 -RUN useradd runpy +RUN useradd runcode # Temporarily needed for pandas RUN echo "deb http://httpredir.debian.org/debian unstable main contrib" >> /etc/apt/sources.list @@ -39,10 +39,10 @@ RUN apt-get -y -o APT::Install-Recommends=0 -o APT::Install-Suggests=0 install \ RUN apt-get clean RUN fc-cache -RUN mkdir -p /opt/runpy -ADD runpy /opt/runpy/ -COPY code_feedback.py /opt/runpy/ -COPY code_runpy_backend.py /opt/runpy/ +RUN mkdir -p /opt/runcode +ADD runcode /opt/runcode/ +COPY code_feedback.py /opt/runcode/ +COPY code_run_backend.py /opt/runcode/ RUN sed -i s/TkAgg/Agg/ /etc/matplotlibrc RUN echo "savefig.dpi : 80" >> /etc/matplotlibrc diff --git a/docker-image-run-py/docker-build.sh b/docker-image-run-py/docker-build.sh index 9f7b73bfc..fdd5e5570 100755 --- a/docker-image-run-py/docker-build.sh +++ b/docker-image-run-py/docker-build.sh @@ -1,5 +1,5 @@ #! /bin/sh -cp ../course/page/code_feedback.py ../course/page/code_runpy_backend.py . +cp ../course/page/code_feedback.py . +cp ../course/page/code_runpy_backend.py code_run_backend.py docker build --no-cache . -t inducer/relate-runpy-amd64 -rm code_feedback.py code_runpy_backend.py - +rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-py/runpy b/docker-image-run-py/runcode similarity index 96% rename from docker-image-run-py/runpy rename to docker-image-run-py/runcode index 02f0b9175..87623061b 100755 --- a/docker-image-run-py/runpy +++ b/docker-image-run-py/runcode @@ -30,18 +30,18 @@ import json import sys import io try: - from code_runpy_backend import Struct, run_code, package_exception + from code_run_backend import Struct, run_code, package_exception except ImportError: try: # When faking a container for unittest - from course.page.code_runpy_backend import ( + from course.page.code_run_backend import ( Struct, run_code, package_exception) except ImportError: # When debugging, i.e., run "python runpy" command line import os sys.path.insert(0, os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir))) - from course.page.code_runpy_backend import ( + from course.page.code_run_backend import ( Struct, run_code, package_exception) from http.server import BaseHTTPRequestHandler diff --git a/setup.cfg b/setup.cfg index d595f7774..d2f7595e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ omit = setup.py local_settings_example.py course/page/code_feedback.py - course/page/code_runpy_backend.py + course/page/code_run*_backend.py */wsgi.py */tests/* */tests.py diff --git a/tests/base_test_mixins.py b/tests/base_test_mixins.py index 169f6a1e2..fb13f0005 100644 --- a/tests/base_test_mixins.py +++ b/tests/base_test_mixins.py @@ -2487,7 +2487,7 @@ def setUpClass(cls): # noqa def setUp(self): super(SubprocessRunpyContainerMixin, self).setUp() self.faked_container_patch = mock.patch( - "course.page.code.SPAWN_CONTAINERS_FOR_RUNPY", False) + "course.page.code.SPAWN_CONTAINERS", False) self.faked_container_patch.start() self.addCleanup(self.faked_container_patch.stop) @@ -2495,9 +2495,9 @@ def setUp(self): def tearDownClass(cls): # noqa super(SubprocessRunpyContainerMixin, cls).tearDownClass() - from course.page.code import SPAWN_CONTAINERS_FOR_RUNPY - # Make sure SPAWN_CONTAINERS_FOR_RUNPY is reset to True - assert SPAWN_CONTAINERS_FOR_RUNPY + from course.page.code import SPAWN_CONTAINERS + # Make sure SPAWN_CONTAINERS is reset to True + assert SPAWN_CONTAINERS if sys.platform.startswith("win"): # Without these lines, tests on Appveyor hanged when all tests # finished. From 146c0aa219f8d5f15de0f02fb7922e709c6e9314 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 19:03:03 -0500 Subject: [PATCH 06/14] Fixed the CodeQuestion blocks. --- course/page/base.py | 6 ++++++ course/page/code.py | 23 ++++++++--------------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/course/page/base.py b/course/page/base.py index e7bbef5a9..5f23b0cf0 100644 --- a/course/page/base.py +++ b/course/page/base.py @@ -768,6 +768,12 @@ def __init__(self, vctx, location, page_desc): if title is None: try: md_body = self.markup_body_for_title() + except AttributeError: + #TODO XXX + from warnings import warn + warn(_("PageBaseWithTitle subclass '%s' does not implement " + "markup_body_for_title()") + % type(self).__name__) except NotImplementedError: from warnings import warn warn(_("PageBaseWithTitle subclass '%s' does not implement " diff --git a/course/page/code.py b/course/page/code.py index 017451e70..0daf120c9 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -56,7 +56,6 @@ class CodeForm(StyledForm): def __init__(self, read_only, interaction_mode, initial_code, language_mode, data=None, *args, **kwargs): super(CodeForm, self).__init__(data, *args, **kwargs) - self.language_mode = language_mode from course.utils import get_codemirror_widget cm_widget, cm_help_text = get_codemirror_widget( @@ -107,11 +106,11 @@ def debug_print(s): def debug_print(s): pass - image = self.container_image + #image = self.container_image #XXX command_path = '/opt/runcode/runcode' user = 'runcode' - if SPAWN_CONTAINERS: + if SPAWN_CONTAINERS and image is not None: docker_url = getattr(settings, "RELATE_DOCKER_URL", "unix://var/run/docker.sock") docker_tls = getattr(settings, "RELATE_DOCKER_TLS_CONFIG", @@ -483,7 +482,6 @@ class CodeQuestion(PageBaseWithTitle, PageBaseWithValue): def __init__(self, vctx, location, page_desc, language_mode): super(CodeQuestion, self).__init__(vctx, location, page_desc) - self.language_mode = language_mode if vctx is not None and hasattr(page_desc, "data_files"): for data_file in page_desc.data_files: @@ -574,6 +572,7 @@ def make_form(self, page_context, page_data, not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), + self.language_mode, answer) else: answer = None @@ -581,6 +580,7 @@ def make_form(self, page_context, page_data, not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), + self.language_mode ) return form @@ -591,6 +591,7 @@ def process_form_post( not page_behavior.may_change_answer, get_editor_interaction_mode(page_context), self._initial_code(), + self.language_mode, post_data, files_data) def answer_data(self, page_context, page_data, form, files_data): @@ -645,7 +646,7 @@ def transfer_attr(name): try: response_dict = request_run_with_retries(run_req, run_timeout=self.page_desc.timeout, - language_mode=self.language_mode) + image=self.container_image) except Exception: from traceback import format_exc response_dict = { @@ -994,10 +995,6 @@ def normalized_bytes_answer(self, page_context, page_data, answer_data): # {{{ python code question -class PythonCodeForm(CodeForm): - def __init__(self, vctx, location, page_desc): - super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc, language_mode='python') - class PythonCodeQuestion(CodeQuestion): """ An auto-graded question allowing an answer consisting of Python code. @@ -1187,7 +1184,7 @@ def suffix(self): return '.py' def __init__(self, vctx, location, page_desc, language_mode='python'): - super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc) + super(PythonCodeQuestion, self).__init__(vctx, location, page_desc, language_mode) # }}} @@ -1395,10 +1392,6 @@ def grade(self, page_context, page_data, answer_data, grade_data): # {{{ octave code question -class OctaveCodeForm(CodeForm): - def __init__(self, vctx, location, page_desc): - super(PythonCodeQuestion, self).__init__(self, vctx, location, page_desc, language_mode='octave') - class OctaveCodeQuestion(CodeQuestion): """ An auto-graded question allowing an answer consisting of Octave code. @@ -1589,7 +1582,7 @@ def suffix(self): return '.m' def __init__(self, vctx, location, page_desc, language_mode='octave'): - super(OctaveCodeQuestion, self).__init__(self, vctx, location, page_desc) + super(OctaveCodeQuestion, self).__init__(vctx, location, page_desc, language_mode) # }}} From 37003018c50350e250f980f7dc4df20a0b744f3f Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 20:53:28 -0500 Subject: [PATCH 07/14] Everything seems to be working properly now. --- course/page/code.py | 4 ++-- docker-image-run-octave/Dockerfile | 3 +-- docker-image-run-octave/runcode | 4 ++-- .../runoc-update/Dockerfile | 2 +- docker-image-run-py/runcode | 2 +- tests/test_pages/test_code.py | 18 +++++++++--------- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/course/page/code.py b/course/page/code.py index 0daf120c9..076736f28 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -149,7 +149,7 @@ def debug_print(s): container_props = docker_cnx.inspect_container(container_id) (port_info,) = (container_props - ["NetworkSettings"]["Ports"]["%d/tcp" % RUNOC_PORT]) + ["NetworkSettings"]["Ports"]["%d/tcp" % CODE_QUESTION_CONTAINER_PORT]) port_host_ip = port_info.get("HostIp") if port_host_ip != "0.0.0.0": @@ -223,7 +223,7 @@ def check_timeout(): start_time = time() debug_print("BEFPOST") - connection.request('POST', '/run-%s'%language_mode, json_run_req, headers) + connection.request('POST', '/run-code', json_run_req, headers) debug_print("AFTPOST") http_response = connection.getresponse() diff --git a/docker-image-run-octave/Dockerfile b/docker-image-run-octave/Dockerfile index 0c9ba5e12..54abbfe61 100644 --- a/docker-image-run-octave/Dockerfile +++ b/docker-image-run-octave/Dockerfile @@ -1,6 +1,6 @@ FROM inducer/debian-amd64-minbase MAINTAINER Neal Davis -EXPOSE 9942 +EXPOSE 9941 RUN useradd runcode # Based on `compdatasci/octave-desktop` Docker image @@ -49,7 +49,6 @@ RUN apt-get update && \ RUN apt-get clean RUN fc-cache -# TODO XXX perhaps I should genericize this as well RUN mkdir -p /opt/runcode ADD runcode /opt/runcode/ COPY code_feedback.py /opt/runcode/ diff --git a/docker-image-run-octave/runcode b/docker-image-run-octave/runcode index bc756f5d2..f3151a138 100755 --- a/docker-image-run-octave/runcode +++ b/docker-image-run-octave/runcode @@ -46,7 +46,7 @@ except ImportError: from http.server import BaseHTTPRequestHandler -PORT = 9942 +PORT = 9941 OUTPUT_LENGTH_LIMIT = 16*1024 TEST_COUNT = 0 @@ -82,7 +82,7 @@ class RunRequestHandler(BaseHTTPRequestHandler): try: print("POST RECEIVED", file=prev_stderr) - if self.path != "/run-octave": + if self.path != "/run-code": raise RuntimeError("unrecognized path in POST") clength = int(self.headers['content-length']) diff --git a/docker-image-run-octave/runoc-update/Dockerfile b/docker-image-run-octave/runoc-update/Dockerfile index cf83a4e6a..a640a3e77 100644 --- a/docker-image-run-octave/runoc-update/Dockerfile +++ b/docker-image-run-octave/runoc-update/Dockerfile @@ -1,6 +1,6 @@ FROM inducer/runoc MAINTAINER Neal Davis -EXPOSE 9942 +EXPOSE 9941 #RUN useradd runcode #RUN apt-get update && \ diff --git a/docker-image-run-py/runcode b/docker-image-run-py/runcode index 87623061b..ea06a70b3 100755 --- a/docker-image-run-py/runcode +++ b/docker-image-run-py/runcode @@ -82,7 +82,7 @@ class RunRequestHandler(BaseHTTPRequestHandler): try: print("POST RECEIVED", file=prev_stderr) - if self.path != "/run-python": + if self.path != "/run-code": raise RuntimeError("unrecognized path in POST") clength = int(self.headers['content-length']) diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index 374deefe6..c35ec347e 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -33,7 +33,7 @@ from course.models import FlowSession from course.page.code import ( - RUNPY_PORT, request_python_run_with_retries, InvalidPingResponse, + CODE_QUESTION_CONTAINER_PORT, request_python_run_with_retries, InvalidPingResponse, is_nuisance_failure, PythonCodeQuestionWithHumanTextFeedback) from course.page.code import ( RUNOC_PORT, request_octave_run_with_retries) @@ -1055,7 +1055,7 @@ def test_docker_container_ping_failure(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1154,7 +1154,7 @@ def test_docker_container_ping_failure(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1203,7 +1203,7 @@ def test_docker_container_ping_return_not_ok(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1243,7 +1243,7 @@ def test_docker_container_runpy_timeout(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1339,7 +1339,7 @@ def test_docker_container_ping_failure(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1438,7 +1438,7 @@ def test_docker_container_ping_failure(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1487,7 +1487,7 @@ def test_docker_container_ping_return_not_ok(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNPY_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} @@ -1527,7 +1527,7 @@ def test_docker_container_runpy_timeout(self): mock_inpect_ctn.return_value = { "NetworkSettings": { - "Ports": {"%d/tcp" % RUNOC_PORT: ( + "Ports": {"%d/tcp" % CODE_QUESTION_CONTAINER_PORT: ( {"HostIp": fake_host_ip, "HostPort": fake_host_port}, )} }} From ef05a7013d81b3ed1da1a582a7aaaa4394d919f0 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 21:04:22 -0500 Subject: [PATCH 08/14] Post davis68 version of docker image. --- docker-image-run-octave/code_feedback.py | 202 ++++++++++++++ docker-image-run-octave/code_run_backend.py | 283 ++++++++++++++++++++ docker-image-run-octave/docker-build.sh | 2 +- local_settings_example.py | 2 +- 4 files changed, 487 insertions(+), 2 deletions(-) create mode 100644 docker-image-run-octave/code_feedback.py create mode 100644 docker-image-run-octave/code_run_backend.py diff --git a/docker-image-run-octave/code_feedback.py b/docker-image-run-octave/code_feedback.py new file mode 100644 index 000000000..416d7b5f8 --- /dev/null +++ b/docker-image-run-octave/code_feedback.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- + +from __future__ import division, print_function + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + + +class GradingComplete(Exception): + pass + + +class Feedback: + def __init__(self): + self.points = None + self.feedback_items = [] + + def set_points(self, points): + self.points = points + + def add_feedback(self, text): + self.feedback_items.append(text) + + def finish(self, points, fb_text): + self.add_feedback(fb_text) + self.set_points(points) + raise GradingComplete() + + def check_numpy_array_sanity(self, name, num_axes, data): + import numpy as np + if not isinstance(data, np.ndarray): + self.finish(0, "'%s' is not a numpy array" % name) + + if isinstance(data, np.matrix): + self.finish(0, "'%s' is a numpy matrix. Do not use those. " + "bit.ly/array-vs-matrix" % name) + + if len(data.shape) != num_axes: + self.finish( + 0, "'%s' does not have the correct number of axes--" + "got: %d, expected: %d" % ( + name, len(data.shape), num_axes)) + + if data.dtype.kind not in "fc": + self.finish( + 0, "'%s' does not consist of floating point numbers--" + "got: '%s'" % (name, data.dtype)) + + def check_numpy_array_features(self, name, ref, data, report_failure=True): + import numpy as np + assert isinstance(ref, np.ndarray) + + def bad(msg): + if report_failure: + self.finish(0, msg) + else: + return False + + if not isinstance(data, np.ndarray): + return bad("'%s' is not a numpy array" % name) + + if isinstance(data, np.matrix): + return bad("'%s' is a numpy matrix. Do not use those. " + "bit.ly/array-vs-matrix" % name) + + if ref.shape != data.shape: + return bad( + "'%s' does not have correct shape--" + "got: '%s', expected: '%s'" % ( + name, data.shape, ref.shape)) + + if ref.dtype.kind != data.dtype.kind: + return bad( + "'%s' does not have correct data type--" + "got: '%s', expected: '%s'" % ( + name, data.dtype, ref.dtype)) + + return True + + def check_numpy_array_allclose(self, name, ref, data, accuracy_critical=True, + rtol=1e-05, atol=1e-08, report_success=True, report_failure=True): + import numpy as np + + if not self.check_numpy_array_features(name, ref, data, report_failure): + return False + + good = np.allclose(ref, data, rtol=rtol, atol=atol) + + if not good: + if report_failure: + self.add_feedback("'%s' is inaccurate" % name) + else: + if report_success: + self.add_feedback("'%s' looks good" % name) + + if accuracy_critical and not good: + self.set_points(0) + raise GradingComplete() + + return good + + def check_list(self, name, ref, data, entry_type=None): + assert isinstance(ref, list) + if not isinstance(data, list): + self.finish(0, "'%s' is not a list" % name) + + if len(ref) != len(data): + self.finish(0, "'%s' has the wrong length--expected %d, got %d" + % (name, len(ref), len(data))) + + if entry_type is not None: + for i, entry in enumerate(data): + if not isinstance(entry, entry_type): + self.finish(0, "'%s[%d]' has the wrong type" % (name, i)) + + def check_scalar(self, name, ref, data, accuracy_critical=True, + rtol=1e-5, atol=1e-8, report_success=True, report_failure=True): + import numpy as np + + if not isinstance(data, (complex, float, int, np.number)): + try: + # Check whether data is a sympy number because sympy + # numbers do not follow the typical interface + # See https://github.com/inducer/relate/pull/284 + if not data.is_number: + self.finish(0, "'%s' is not a number" % name) + except AttributeError: + self.finish(0, "'%s' is not a number" % name) + + good = False + + if rtol is not None and abs(ref-data) < abs(ref)*rtol: + good = True + if atol is not None and abs(ref-data) < atol: + good = True + + if not good: + if report_failure: + self.add_feedback("'%s' is inaccurate" % name) + else: + if report_success: + self.add_feedback("'%s' looks good" % name) + + if accuracy_critical and not good: + self.set_points(0) + raise GradingComplete() + + return good + + def call_user(self, f, *args, **kwargs): + try: + return f(*args, **kwargs) + except Exception: + if callable(f): + try: + callable_name = f.__name__ + except Exception as e_name: + callable_name = ( + "" + % ( + type(e_name).__name__, + str(e_name))) + from traceback import format_exc + self.add_feedback( + "

    " + "The callable '%s' supplied in your code failed with " + "an exception while it was being called by the grading " + "code:" + "

    " + "
    %s
    " + % ( + callable_name, + "".join(format_exc()))) + else: + self.add_feedback( + "

    " + "Your code was supposed to supply a function or " + "callable, but the variable you supplied was not " + "callable." + "

    ") + + self.set_points(0) + raise GradingComplete() diff --git a/docker-image-run-octave/code_run_backend.py b/docker-image-run-octave/code_run_backend.py new file mode 100644 index 000000000..525f7dff7 --- /dev/null +++ b/docker-image-run-octave/code_run_backend.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import sys +import traceback + +try: + from .code_feedback import Feedback, GradingComplete +except SystemError: + from code_feedback import Feedback, GradingComplete # type: ignore +except ImportError: + from code_feedback import Feedback, GradingComplete # type: ignore + + +__doc__ = """ +PROTOCOL +======== + +.. class:: Request + + .. attribute:: setup_code + + .. attribute:: names_for_user + + .. attribute:: user_code + + .. attribute:: names_from_user + + .. attribute:: test_code + + .. attribute:: data_files + + A dictionary from data file names to their + base64-cencoded contents. + Optional. + + .. attribute:: compile_only + + :class:`bool` + +.. class Response:: + .. attribute:: result + + One of + + * ``success`` + * ``timeout`` + * ``uncaught_error`` + * ``setup_compile_error`` + * ``setup_error``, + * ``user_compile_error`` + * ``user_error`` + * ``test_compile_error`` + * ``test_error`` + + Always present. + + .. attribute:: message + + Optional. + + .. attribute:: traceback + + Optional. + + .. attribute:: stdout + + Whatever came out of stdout. + + Optional. + + .. attribute:: stderr + + Whatever came out of stderr. + + Optional. + + .. attribute:: figures + + A list of ``(index, mime_type, string)``, where *string* is a + base64-encoded representation of the figure. *index* will usually + correspond to the matplotlib figure number. + + Optional. + + .. attribute:: html + + A list of HTML strings generated. These are aggressively sanitized + before being rendered. + + .. attribute:: points + + A number between 0 and 1 (inclusive). + + Present on ``success`` if :attr:`Request.compile_only` is *False*. + + .. attribute:: feedback + + A list of strings. + + Present on ``success`` if :attr:`Request.compile_only` is *False*. +""" + + +# {{{ tools + +class Struct(object): + def __init__(self, entries): + for name, val in entries.items(): + self.__dict__[name] = val + + def __repr__(self): + return repr(self.__dict__) + +# }}} + + +def substitute_correct_code_into_test_code(test_code, correct_code): + import re + CORRECT_CODE_TAG = re.compile(r"^(\s*)###CORRECT_CODE###\s*$") # noqa + + new_test_code_lines = [] + for l in test_code.split("\n"): + match = CORRECT_CODE_TAG.match(l) + if match is not None: + prefix = match.group(1) + for cc_l in correct_code.split("\n"): + new_test_code_lines.append(prefix+cc_l) + else: + new_test_code_lines.append(l) + + return "\n".join(new_test_code_lines) + + +def package_exception(result, what): + tp, val, tb = sys.exc_info() + result["result"] = what + result["message"] = "%s: %s" % (tp.__name__, str(val)) + result["traceback"] = "".join( + traceback.format_exception(tp, val, tb)) + + +def run_code(result, run_req): + # {{{ set up octave process + + import oct2py + + oc = oct2py.Oct2Py() + + # }}} + + # {{{ run code + + data_files = {} + if hasattr(run_req, "data_files"): + from base64 import b64decode + for name, contents in run_req.data_files.items(): + # This part "cheats" a litle, since Octave lets us evaluate functions + # in the same context as the main code. (MATLAB segregates these.) + data_files[name] = b64decode(contents.encode()) + oc.eval(b64decode(contents.encode()).decode("utf-8")) + + generated_html = [] + result["html"] = generated_html + + def output_html(s): + generated_html.append(s) + + feedback = Feedback() + maint_ctx = { + "feedback": feedback, + "user_code": run_req.user_code, + "data_files": data_files, + "output_html": output_html, + "GradingComplete": GradingComplete, + } + + if run_req.setup_code is not None: + try: + oc.eval(run_req.setup_code) + except Exception: + package_exception(result, "setup_error") + return + + ''' + user_ctx = {} + if hasattr(run_req, "names_for_user"): #XXX unused for Octave context currently + for name in run_req.names_for_user: + if name not in maint_ctx: + result["result"] = "setup_error" + result["message"] = "Setup code did not define '%s'." % name + + user_ctx[name] = maint_ctx[name] + + from copy import deepcopy + user_ctx = deepcopy(user_ctx) + ''' + + try: + oc.eval(run_req.user_code) + except Exception: + package_exception(result, "user_error") + return + + # {{{ export plots + + ''' + if "matplotlib" in sys.modules: + import matplotlib.pyplot as pt + from io import BytesIO + from base64 import b64encode + + format = "png" + mime = "image/png" + figures = [] + + for fignum in pt.get_fignums(): + pt.figure(fignum) + bio = BytesIO() + try: + pt.savefig(bio, format=format) + except Exception: + pass + else: + figures.append( + (fignum, mime, b64encode(bio.getvalue()).decode())) + + result["figures"] = figures + ''' + # }}} + + if hasattr(run_req, "names_from_user"): + values = [] + for name in run_req.names_from_user: + try: + maint_ctx[name] = oc.pull(name) + except oct2py.Oct2PyError: + feedback.add_feedback( + "Required answer variable '%s' is not defined." + % name) + maint_ctx[name] = None + + if run_req.test_code is not None: # XXX test code is written in Python + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.test_code + exec(run_req.test_code, maint_ctx) + except GradingComplete: + pass + except Exception: + package_exception(result, "test_error") + return + + result["points"] = feedback.points + result["feedback"] = feedback.feedback_items + + # }}} + + result["result"] = "success" + +# vim: foldmethod=marker diff --git a/docker-image-run-octave/docker-build.sh b/docker-image-run-octave/docker-build.sh index dbcf6b1de..eaabf1669 100755 --- a/docker-image-run-octave/docker-build.sh +++ b/docker-image-run-octave/docker-build.sh @@ -1,5 +1,5 @@ #! /bin/sh cp ../course/page/code_feedback.py . cp ../course/page/code_runoc_backend.py code_run_backend.py -docker build --no-cache . -t inducer/runoc +docker build --no-cache . -t davis68/runoc rm code_feedback.py code_run_backend.py diff --git a/local_settings_example.py b/local_settings_example.py index 75ddeeb39..8b13f89fb 100644 --- a/local_settings_example.py +++ b/local_settings_example.py @@ -343,7 +343,7 @@ # A string containing the image ID of the docker image to be used to run # student Python code. Docker should download the image on first run. RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64" -RELATE_DOCKER_RUNOC_IMAGE = "inducer/runoc" +RELATE_DOCKER_RUNOC_IMAGE = "davis68/runoc" # RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64-tensorflow" # (bigger, but includes TensorFlow) From 8bfe887aff18aad7a2cf2fa7fb3a2f85ed6392da Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 22:24:00 -0500 Subject: [PATCH 09/14] Post changes to tests. --- .gitignore | 3 + docker-image-run-octave/docker-build.sh | 2 +- docker-image-run-py/Dockerfile | 3 + exercise-docker.py | 4 +- local_settings_example.py | 2 +- tests/test_pages/test_code.py | 78 ++++++++++++------------- 6 files changed, 49 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 3fc425ee2..3231659ce 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ git-roots /.idea /.env + +my_todo +password diff --git a/docker-image-run-octave/docker-build.sh b/docker-image-run-octave/docker-build.sh index eaabf1669..b7e7d3d6e 100755 --- a/docker-image-run-octave/docker-build.sh +++ b/docker-image-run-octave/docker-build.sh @@ -1,5 +1,5 @@ #! /bin/sh cp ../course/page/code_feedback.py . cp ../course/page/code_runoc_backend.py code_run_backend.py -docker build --no-cache . -t davis68/runoc +docker build --no-cache . -t davis68/relate-octave rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-py/Dockerfile b/docker-image-run-py/Dockerfile index 6f620d2ec..fa0e460ae 100644 --- a/docker-image-run-py/Dockerfile +++ b/docker-image-run-py/Dockerfile @@ -11,6 +11,7 @@ RUN echo 'APT::Default-Release "testing";' >> /etc/apt/apt.conf RUN apt-get update RUN apt-get -y -o APT::Install-Recommends=0 -o APT::Install-Suggests=0 install \ python3-scipy \ + python3-numpy \ python3-pip \ python3-matplotlib \ python3-pillow \ @@ -39,6 +40,8 @@ RUN apt-get -y -o APT::Install-Recommends=0 -o APT::Install-Suggests=0 install \ RUN apt-get clean RUN fc-cache +RUN pip3 install numpy==1.16.1 + RUN mkdir -p /opt/runcode ADD runcode /opt/runcode/ COPY code_feedback.py /opt/runcode/ diff --git a/exercise-docker.py b/exercise-docker.py index 09821f7b6..bc91ada30 100644 --- a/exercise-docker.py +++ b/exercise-docker.py @@ -1,5 +1,5 @@ def main(): - from course.page import request_python_run + from course.page import request_run req = { "setup_code": "a,b=5,2", @@ -13,7 +13,7 @@ def main(): while True: print count count += 1 - res = request_python_run(req, 5, + res = request_run(req, 5, image="inducer/runpy-i386") if res["result"] != "success": print res diff --git a/local_settings_example.py b/local_settings_example.py index 8b13f89fb..dc4e9d016 100644 --- a/local_settings_example.py +++ b/local_settings_example.py @@ -343,7 +343,7 @@ # A string containing the image ID of the docker image to be used to run # student Python code. Docker should download the image on first run. RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64" -RELATE_DOCKER_RUNOC_IMAGE = "davis68/runoc" +RELATE_DOCKER_RUNOC_IMAGE = "davis68/relate-octave" # RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64-tensorflow" # (bigger, but includes TensorFlow) diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index c35ec347e..cbfd41d0d 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -33,7 +33,7 @@ from course.models import FlowSession from course.page.code import ( - CODE_QUESTION_CONTAINER_PORT, request_python_run_with_retries, InvalidPingResponse, + CODE_QUESTION_CONTAINER_PORT, request_run_with_retries, InvalidPingResponse, is_nuisance_failure, PythonCodeQuestionWithHumanTextFeedback) from course.page.code import ( RUNOC_PORT, request_octave_run_with_retries) @@ -75,7 +75,7 @@ "The grading code failed. Sorry about that." ) -RUNPY_WITH_RETRIES_PATH = "course.page.code.request_python_run_with_retries" +RUNPY_WITH_RETRIES_PATH = "course.page.code.request_run_with_retries" AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN = ( "'correctness' is invalid: expecting " @@ -411,13 +411,13 @@ def test_question_with_human_feedback_value_0_feedback_0_percentage(self): self.assertSandboxHasValidPage(resp) self.assertSandboxWarningTextContain(resp, None) - def test_request_python_run_with_retries_raise_uncaught_error_in_sandbox(self): + def test_request_run_with_retries_raise_uncaught_error_in_sandbox(self): with mock.patch( RUNPY_WITH_RETRIES_PATH, autospec=True ) as mock_runpy: expected_error_str = ("This is an error raised with " - "request_python_run_with_retries") + "request_run_with_retries") # correct_code_explanation and correct_code expected_feedback = ( @@ -438,13 +438,13 @@ def test_request_python_run_with_retries_raise_uncaught_error_in_sandbox(self): # No email when in sandbox self.assertEqual(len(mail.outbox), 0) - def test_request_python_run_with_retries_raise_uncaught_error_debugging(self): + def test_request_run_with_retries_raise_uncaught_error_debugging(self): with mock.patch( RUNPY_WITH_RETRIES_PATH, autospec=True ) as mock_runpy: expected_error_str = ("This is an error raised with " - "request_python_run_with_retries") + "request_run_with_retries") mock_runpy.side_effect = RuntimeError(expected_error_str) with override_settings(DEBUG=True): @@ -457,13 +457,13 @@ def test_request_python_run_with_retries_raise_uncaught_error_debugging(self): # No email when debugging self.assertEqual(len(mail.outbox), 0) - def test_request_python_run_with_retries_raise_uncaught_error(self): + def test_request_run_with_retries_raise_uncaught_error(self): with mock.patch( RUNPY_WITH_RETRIES_PATH, autospec=True ) as mock_runpy: expected_error_str = ("This is an error raised with " - "request_python_run_with_retries") + "request_run_with_retries") mock_runpy.side_effect = RuntimeError(expected_error_str) with mock.patch("course.page.PageContext") as mock_page_context: @@ -484,13 +484,13 @@ def test_request_python_run_with_retries_raise_uncaught_error(self): self.assertEqual(len(mail.outbox), 1) self.assertIn(expected_error_str, mail.outbox[0].body) - def test_send_email_failure_when_request_python_run_with_retries_raise_uncaught_error(self): # noqa + def test_send_email_failure_when_request_run_with_retries_raise_uncaught_error(self): # noqa with mock.patch( RUNPY_WITH_RETRIES_PATH, autospec=True ) as mock_runpy: expected_error_str = ("This is an error raised with " - "request_python_run_with_retries") + "request_run_with_retries") mock_runpy.side_effect = RuntimeError(expected_error_str) with mock.patch("course.page.PageContext") as mock_page_context: @@ -548,7 +548,7 @@ def assert_runpy_result_and_response(self, result_type, expected_msgs=None, correctness) self.assertEqual(len(mail.outbox), mail_count) - def test_request_python_run_with_retries_timed_out(self): + def test_request_run_with_retries_timed_out(self): self.assert_runpy_result_and_response( "timeout", "Your code took too long to execute.") @@ -1003,7 +1003,7 @@ def test_feedback_code_error_exceed_max_extra_credit_factor_email(self): class RequestPythonRunWithRetriesTest(unittest.TestCase): - # Testing course.page.code.request_python_run_with_retries, + # Testing course.page.code.request_run_with_retries, # adding tests for use cases that didn't cover in other tests @override_settings(RELATE_DOCKER_RUNPY_IMAGE="some_other_image") @@ -1015,7 +1015,7 @@ def test_image_none(self): mock_create_ctn.return_value = {} with self.assertRaises(KeyError): - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1) self.assertEqual(mock_create_ctn.call_count, 1) self.assertIn("some_other_image", mock_create_ctn.call_args[0]) @@ -1031,7 +1031,7 @@ def test_image_not_none(self): my_image = "my_runpy_image" with self.assertRaises(KeyError): - request_python_run_with_retries( + request_run_with_retries( run_req={}, image=my_image, run_timeout=0.1) self.assertEqual(mock_create_ctn.call_count, 1) self.assertIn(my_image, mock_create_ctn.call_args[0]) @@ -1067,7 +1067,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1083,7 +1083,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1100,7 +1100,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1116,7 +1116,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1135,12 +1135,12 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): with self.assertRaises(socket_error) as e: - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(e.exception.errno, my_socket_error.errno) with self.assertRaises(socket_error) as e: - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(e.exception.errno, my_socket_error.errno) @@ -1170,7 +1170,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1215,7 +1215,7 @@ def test_docker_container_ping_return_not_ok(self): mock_ctn_request.side_effect = lambda x, y: None mock_ctn_get_response.return_value = six.BytesIO(b"NOT OK") - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1255,7 +1255,7 @@ def test_docker_container_runpy_timeout(self): mock_ctn_request.side_effect = [None, sock_timeout] mock_ctn_get_response.return_value = six.BytesIO(b"OK") - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "timeout") self.assertEqual(res["exec_host"], fake_host_ip) @@ -1263,13 +1263,13 @@ def test_docker_container_runpy_timeout(self): @skipIf(six.PY2, "PY2 doesn't support subTest") def test_docker_container_runpy_retries_count(self): with ( - mock.patch("course.page.code.request_python_run")) as mock_req_run, ( # noqa + mock.patch("course.page.code.request_run")) as mock_req_run, ( # noqa mock.patch("course.page.code.is_nuisance_failure")) as mock_is_nuisance_failure: # noqa expected_result = "this is my custom result" mock_req_run.return_value = {"result": expected_result} with self.subTest(actual_retry_count=4): mock_is_nuisance_failure.side_effect = [True, True, True, False] - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=5) self.assertEqual(res["result"], expected_result) self.assertEqual(mock_req_run.call_count, 4) @@ -1279,7 +1279,7 @@ def test_docker_container_runpy_retries_count(self): mock_is_nuisance_failure.reset_mock() with self.subTest(actual_retry_count=2): mock_is_nuisance_failure.side_effect = [True, True, True, False] - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=1) self.assertEqual(res["result"], expected_result) self.assertEqual(mock_req_run.call_count, 2) @@ -1299,7 +1299,7 @@ def test_image_none(self): mock_create_ctn.return_value = {} with self.assertRaises(KeyError): - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1) self.assertEqual(mock_create_ctn.call_count, 1) self.assertIn("some_other_image", mock_create_ctn.call_args[0]) @@ -1315,7 +1315,7 @@ def test_image_not_none(self): my_image = "my_runoc_image" with self.assertRaises(KeyError): - request_python_run_with_retries( + request_run_with_retries( run_req={}, image=my_image, run_timeout=0.1) self.assertEqual(mock_create_ctn.call_count, 1) self.assertIn(my_image, mock_create_ctn.call_args[0]) @@ -1351,7 +1351,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1367,7 +1367,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1384,7 +1384,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1400,7 +1400,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1419,12 +1419,12 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): with self.assertRaises(socket_error) as e: - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(e.exception.errno, my_socket_error.errno) with self.assertRaises(socket_error) as e: - request_python_run_with_retries( + request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(e.exception.errno, my_socket_error.errno) @@ -1454,7 +1454,7 @@ def test_docker_container_ping_failure(self): # force timeout with mock.patch("course.page.code.DOCKER_TIMEOUT", 0.0001): - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1499,7 +1499,7 @@ def test_docker_container_ping_return_not_ok(self): mock_ctn_request.side_effect = lambda x, y: None mock_ctn_get_response.return_value = six.BytesIO(b"NOT OK") - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "uncaught_error") self.assertEqual(res['message'], @@ -1539,7 +1539,7 @@ def test_docker_container_runpy_timeout(self): mock_ctn_request.side_effect = [None, sock_timeout] mock_ctn_get_response.return_value = six.BytesIO(b"OK") - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=0) self.assertEqual(res["result"], "timeout") self.assertEqual(res["exec_host"], fake_host_ip) @@ -1553,7 +1553,7 @@ def test_docker_container_runoc_retries_count(self): mock_req_run.return_value = {"result": expected_result} with self.subTest(actual_retry_count=4): mock_is_nuisance_failure.side_effect = [True, True, True, False] - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=5) self.assertEqual(res["result"], expected_result) self.assertEqual(mock_req_run.call_count, 4) @@ -1563,7 +1563,7 @@ def test_docker_container_runoc_retries_count(self): mock_is_nuisance_failure.reset_mock() with self.subTest(actual_retry_count=2): mock_is_nuisance_failure.side_effect = [True, True, True, False] - res = request_python_run_with_retries( + res = request_run_with_retries( run_req={}, run_timeout=0.1, retry_count=1) self.assertEqual(res["result"], expected_result) self.assertEqual(mock_req_run.call_count, 2) From a4cedefbf9a0ccbce885186af8a80754481640f2 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 2 Apr 2019 22:24:14 -0500 Subject: [PATCH 10/14] Post changes to tests. --- tests/test_pages/markdowns.py | 365 +++++++++++++++++++++++++++++++++- 1 file changed, 364 insertions(+), 1 deletion(-) diff --git a/tests/test_pages/markdowns.py b/tests/test_pages/markdowns.py index dc0039d84..336384b57 100644 --- a/tests/test_pages/markdowns.py +++ b/tests/test_pages/markdowns.py @@ -7,7 +7,7 @@ # }}} -# {{{ code questions +# {{{ python code questions CODE_MARKDWON = """ type: PythonCodeQuestion @@ -418,4 +418,367 @@ # }}} +# {{{ octave code questions + +OCTAVE_CODE_MARKDWON = """ +type: OctaveCodeQuestion +access_rules: + add_permissions: + - change_answer +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding 1 and 2, and assign it to c + +names_from_user: [c] + +initial_code: | + c = + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = 3 + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = 2 + 1 + +correct_code_explanation: This is the [explanation](http://example.com/1). +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +data_files: + - question-data/random-data.npy + %(extra_data_file)s +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_WITH_DATAFILES_BAD_FORMAT = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +data_files: + - question-data/random-data.npy + - - foo + - bar +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 = """ +type: OctaveCodeQuestion +access_rules: + add_permissions: + - see_answer_after_submission +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 = """ +type: OctaveCodeQuestion +access_rules: + remove_permissions: + - see_answer_after_submission +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_TEST_CODE = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +correct_code: | + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_CORRECT_CODE = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +""" + +OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(%(full_points)s, "Your computed c was correct.") + else: + feedback.finish(%(min_points)s, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" # noqa + +OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Python + +setup_code: | + import random + + a = random.uniform(-10, 10) + b = random.uniform(-10, 10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(%(full_points)s, "Your computed c was correct.") + else: + feedback.finish(%(min_points)s, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" # noqa + +# }}} + # vim: fdm=marker From 1e6051b20d1b6ede2722382f4bc43e451f4d6152 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 9 Apr 2019 08:36:45 -0500 Subject: [PATCH 11/14] Post tests. --- tests/test_pages/markdowns.py | 76 ++++----- tests/test_pages/test_code.py | 300 ++++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+), 47 deletions(-) diff --git a/tests/test_pages/markdowns.py b/tests/test_pages/markdowns.py index 336384b57..ed0693cb9 100644 --- a/tests/test_pages/markdowns.py +++ b/tests/test_pages/markdowns.py @@ -462,17 +462,15 @@ value: 1 timeout: 10 data_files: - - question-data/random-data.npy + - question-data/random-data.m %(extra_data_file)s prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -501,18 +499,16 @@ value: 1 timeout: 10 data_files: - - question-data/random-data.npy + - question-data/random-data.m - - foo - bar prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -545,13 +541,11 @@ timeout: 10 prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -584,13 +578,11 @@ timeout: 10 prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -621,13 +613,11 @@ single_submission: True prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -657,13 +647,11 @@ single_submission: True prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -681,13 +669,11 @@ single_submission: True prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -714,13 +700,11 @@ timeout: 10 prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] @@ -750,13 +734,11 @@ timeout: 10 prompt: | - # Adding two numbers in Python + # Adding two numbers in Octave setup_code: | - import random - - a = random.uniform(-10, 10) - b = random.uniform(-10, 10) + a = unifrnd(-10,10) + b = unifrnd(-10,10) names_for_user: [a, b] diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index cbfd41d0d..a83179e6b 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -1001,6 +1001,306 @@ def test_feedback_code_error_exceed_max_extra_credit_factor_email(self): # }}} + # {{{ Octave code tests patterned after Python tests + + def test_data_files_missing_random_question_data_file(self): + file_name = "foo" + markdown = ( + markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + % {"extra_data_file": "- %s" % file_name} + ) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxNotHasValidPage(resp) + self.assertResponseContextContains( + resp, PAGE_ERRORS, "data file '%s' not found" % file_name) + + def test_data_files_missing_random_question_data_file_bad_format(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_WITH_DATAFILES_BAD_FORMAT + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxNotHasValidPage(resp) + self.assertResponseContextContains( + resp, PAGE_ERRORS, "data file '%s' not found" % "['foo', 'bar']") + + def test_not_multiple_submit_warning(self): + markdown = ( + markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + % {"extra_data_file": ""} + ) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_not_multiple_submit_warning2(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_not_multiple_submit_warning3(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_allow_multiple_submit(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + def test_explicity_not_allow_multiple_submit(self): + markdown = ( + markdowns.OCTAVE_CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT + % {"extra_data_file": ""} + ) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + def test_question_without_test_code(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_TEST_CODE + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, NO_CORRECTNESS_INFO_MSG) + + def test_question_without_correct_code(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_CORRECT_CODE + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + + def test_feedback_points_close_to_1(self): + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1.000000000002, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + + def test_feedback_code_exceed_1(self): + feedback_points = 1.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1.1) + + expected_feedback = "Your answer is correct and earned bonus points." + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, expected_feedback) + + def test_feedback_code_positive_close_to_0(self): + # https://github.com/inducer/relate/pull/448#issuecomment-363655132 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": 0.00000000001 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 0) + + def test_feedback_code_negative_close_to_0(self): + # https://github.com/inducer/relate/pull/448#issuecomment-363655132 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": -0.00000000001 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 0) + + def test_feedback_code_error_close_below_max_auto_feedback_points(self): + feedback_points = MAX_EXTRA_CREDIT_FACTOR - 1e-6 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals( + resp, MAX_EXTRA_CREDIT_FACTOR) + + def test_feedback_code_error_close_above_max_auto_feedback_points(self): + feedback_points = MAX_EXTRA_CREDIT_FACTOR + 1e-6 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals( + resp, MAX_EXTRA_CREDIT_FACTOR) + + def test_feedback_code_error_negative_feedback_points(self): + invalid_feedback_points = -0.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": invalid_feedback_points + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + + def test_feedback_code_error_exceed_max_extra_credit_factor(self): + invalid_feedback_points = 10.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": invalid_feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + + def test_feedback_code_error_exceed_max_extra_credit_factor_email(self): + invalid_feedback_points = 10.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": invalid_feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + with mock.patch("course.page.PageContext") as mock_page_context: + mock_page_context.return_value.in_sandbox = False + mock_page_context.return_value.course = self.course + + # This remove the warning caused by mocked commit_sha value + # "CacheKeyWarning: Cache key contains characters that + # will cause errors ..." + mock_page_context.return_value.commit_sha = b"1234" + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + self.assertEqual(len(mail.outbox), 1) + + self.assertIn(error_msg, mail.outbox[0].body) + + # }}} class RequestPythonRunWithRetriesTest(unittest.TestCase): # Testing course.page.code.request_run_with_retries, From cd7cfca4b14619f521786be387c11e254dab8827 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 9 Apr 2019 09:06:33 -0500 Subject: [PATCH 12/14] Fix some imports from older version of code. --- tests/test_pages/test_code.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index a83179e6b..434c48b76 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -35,8 +35,6 @@ from course.page.code import ( CODE_QUESTION_CONTAINER_PORT, request_run_with_retries, InvalidPingResponse, is_nuisance_failure, PythonCodeQuestionWithHumanTextFeedback) -from course.page.code import ( - RUNOC_PORT, request_octave_run_with_retries) from course.utils import FlowPageContext, CoursePageContext from course.constants import MAX_EXTRA_CREDIT_FACTOR From c477cd0b33e22b0a9a1ac0c45346c93abae4f1c6 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Tue, 9 Apr 2019 09:19:19 -0500 Subject: [PATCH 13/14] Fix some imports from older version of code. --- tests/test_pages/test_code.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index 434c48b76..9383aac89 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -1585,7 +1585,7 @@ def test_docker_container_runpy_retries_count(self): class RequestOctaveRunWithRetriesTest(unittest.TestCase): - # Testing course.page.code.request_octave_run_with_retries, + # Testing course.page.code.request_run_with_retries, # adding tests for use cases that didn't cover in other tests @override_settings(RELATE_DOCKER_RUNOC_IMAGE="some_other_image") @@ -1845,7 +1845,7 @@ def test_docker_container_runpy_timeout(self): @skipIf(six.PY2, "PY2 doesn't support subTest") def test_docker_container_runoc_retries_count(self): with ( - mock.patch("course.page.code.request_octave_run")) as mock_req_run, ( # noqa + mock.patch("course.page.code.request_run")) as mock_req_run, ( # noqa mock.patch("course.page.code.is_nuisance_failure")) as mock_is_nuisance_failure: # noqa expected_result = "this is my custom result" mock_req_run.return_value = {"result": expected_result} From 8c2d4f00bf22ef5c53b9703c975e88e4f3b6dc69 Mon Sep 17 00:00:00 2001 From: Neal Davis Date: Thu, 11 Apr 2019 12:01:51 -0500 Subject: [PATCH 14/14] Turn off debugging. --- course/page/code.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/course/page/code.py b/course/page/code.py index 076736f28..765fd302e 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -98,7 +98,7 @@ def request_run(run_req, run_timeout, image=None): import errno from docker.errors import APIError as DockerAPIError - debug = True + debug = False if debug: def debug_print(s): print(s)