Merge branch 'main' of github.com:expfactory/expfactory-deploy

.github/workflows/ci.yml

@@ -21,18 +21,18 @@ jobs:
     steps:
 
       - name: Checkout Code Repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python 3.8
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v3.1.0
         with:
           python-version: 3.8
 
       # Run all pre-commit hooks on all the files.
       # Getting only staged files can be tricky in case a new PR is opened
       # since the action is run on a branch in detached head state
       - name: Install and Run Pre-commit 
-        uses: pre-commit/[email protected].0
+        uses: pre-commit/[email protected].3
 
   # With no caching at all the entire ci process takes 4m 30s to complete!
 #  pytest:
@@ -41,7 +41,7 @@ jobs:
 #    steps:
 #
 #      - name: Checkout Code Repository
-#        uses: actions/checkout@v2
+#        uses: actions/checkout@v3
 #
 #      - name: Build the Stack
 #        run:  docker-compose -f local.yml build

README.rst

@@ -32,4 +32,4 @@ About
 This repository is intended as a replacement for
 `expfactory-docker<https://github.com/expfactory/expfactory-docker>`_ and
 seeks to expand on the ease of use of the first iteration of expfactory and
-integrate the tenets of reproducibility from the second iteration.
+integrate the tenets of reproducibility from the second iteration. 

expfactory_deploy/experiments/forms.py

@@ -14,31 +14,40 @@
 from giturlparse import parse
 from experiments import models as models
 
-class ExperimentUploadForm(forms.Form):
-    url = forms.URLField()
+class RepoOriginForm(ModelForm):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.helper = FormHelper(self)
+        self.helper.add_input(Submit('submit', 'Submit'))
+
+    class Meta:
+        model  = models.RepoOrigin
+        fields = ["url"]
+        widgets = {
+            "url": forms.TextInput(),
+        }
+
+    # Todo: extract user/org name and use them in path and name 
+    # could also imagine adding index for collisions.
     def clean(self):
         cleaned_data = super().clean()
         url = cleaned_data["url"]
         gitparse_url = parse(url)
         if not gitparse_url.valid:
             self.add_error("url", "gitparseurl library could not validate repo url.")
-        url = gitparse_url.urls.get('https', url)
-        self.cleaned_data.update({'url', url})
-
-        path = pathlib.Path(settings.REPO_DIR, gitparse_url)
-        path.mkdir(parents=True, exists_ok=True)
-        try:
-            models.RepoOrigin.get(origin=url)
-            self.add_error("url", f"repository with this origin already exists")
-        except models.RepoOrigin.DoexNotExist:
-            db_repo_origin = models.RepoOrigin(origin=url, path=path)
+        else:
+            name = gitparse_url.name
+            path = pathlib.Path(settings.REPO_DIR, name)
+            path.mkdir(parents=True, exist_ok=True)
+            self.instance.path = path
+            self.instance.name = name
             try:
-                repo = git.Repo.clone_from(url, path)
-                db_repo_origin.save()
-            except git.exc.GitError as e:
-                self.add_error("url", e)
-
+                models.RepoOrigin.objects.get(name=name, path=path)
+            except ObjectDoesNotExist:
+                pass
+            else:
+                self.add_error('Repository with this name currently exists')
         return cleaned_data
 
 class SubjectCount(forms.Form):
@@ -117,7 +126,7 @@ def clean(self):
         # should we return gitpython error message here?
         if not exp_instance.origin.is_valid_commit(commit):
             raise forms.ValidationError(
-                f"Commit '{commit}' is invalid for {exp_instance.origin}"
+                f"Commit '{commit}' is invalid for {exp_instance.url}"
             )
 
         cleaned_data["commit"] = commit

expfactory_deploy/experiments/migrations/0025_auto_20220331_1715.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.7 on 2022-03-31 17:15
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('experiments', '0024_auto_20220330_1808'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='repoorigin',
+            old_name='origin',
+            new_name='url',
+        ),
+    ]

expfactory_deploy/experiments/migrations/0026_auto_20220331_1752.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.7 on 2022-03-31 17:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('experiments', '0025_auto_20220331_1715'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='repoorigin',
+            name='url',
+            field=models.TextField(unique=True),
+        ),
+    ]

expfactory_deploy/experiments/models.py

@@ -7,7 +7,9 @@
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
+from django.dispatch import receiver
 from django.urls import reverse
+from giturlparse import parse
 from model_utils import Choices
 from model_utils.fields import MonitorField, StatusField
 from model_utils.models import StatusModel, TimeStampedModel
@@ -48,12 +50,12 @@ class Meta:
 class RepoOrigin(models.Model):
     """ Location of a repository that contains an experiment """
 
-    origin = models.URLField(unique=True)
-    path = models.TextField()
+    url = models.TextField(unique=True)
+    path = models.TextField(unique=True)
     name = models.TextField(blank=True, unique=True)
 
     def __str__(self):
-        return self.origin
+        return self.url
 
     def get_latest_commit(self):
         return repo.get_latest_commit(self.path)
@@ -87,6 +89,16 @@ def update_dependents(self):
             battexp.experiment_instance = new_instance
             battexp.save()
 
+    def clone(self):
+        repo = git.Repo.clone_from(self.url, self.path)
+
+
+
+''' Will likely want to have clone be called as a task from here
+@receiver(models.signals.post_save, sender=RepoOrigin)
+def execute_after_save(sender, instance, created, *args, **kwargs):
+    if created:
+'''
 
 @reversion.register()
 class ExperimentRepo(models.Model):
@@ -113,11 +125,11 @@ def get_latest_commit(self):
     def url(self):
         base_path = self.origin.path
         exp_location = self.location
-        if "[email protected]:" in self.origin.origin:
-            origin_url = self.origin.origin.replace("[email protected]:", "https://github.com/")
+        if "[email protected]:" in self.origin.url:
+            origin_url = self.origin.url.replace("[email protected]:", "https://github.com/")
             exp_location = self.location.replace(base_path, f"/tree/{self.branch}")
         else:
-            origin_url = self.origin.origin
+            origin_url = self.origin.url
         return f"{origin_url}{exp_location}"
 
     def __str__(self):

expfactory_deploy/experiments/urls.py

@@ -22,6 +22,11 @@
         views.add_new_experiments,
         name="experiment-repo-create",
     ),
+    path(
+        "repo/add/",
+        views.RepoOriginCreate.as_view(),
+        name="repo-origin-create",
+    ),
     path(
         "experiment_repo/<int:pk>/update/",
         views.ExperimentRepoUpdate.as_view(),

expfactory_deploy/experiments/utils/repo.py

@@ -13,13 +13,18 @@
 
 
 def find_valid_dirs(repo):
-    fp = open(pathlib.Path(__file__).parent.joinpath("experiment_schema.json"))
-    schema = json.load(fp)
+    with open(pathlib.Path(__file__).parent.joinpath("experiment_schema.json")) as fp:
+        schema = json.load(fp)
     valid_dirs = []
     errors = []
+    # switch to scandir if too slow
     for root, dir, files in os.walk(repo):
         if "config.json" not in files:
             continue
+
+        if "config.json" in files and "index.html" in files:
+            # expfactory2 - todo
+            continue
         with open(os.path.join(root, "config.json")) as config_fp:
             config = json.load(config_fp)
             try:

expfactory_deploy/experiments/views.py

@@ -21,6 +21,21 @@
 
 from expfactory_deploy_local.utils import generate_experiment_context
 
+# Repo Views
+
+class RepoOriginListView(ListView):
+    model = models.RepoOrigin
+    queryset = models.RepoOrigin.objects.prefetch_related("experimentrepo_set")
+
+class RepoOriginCreate(CreateView):
+    model = models.RepoOrigin
+    form_class = forms.RepoOriginForm
+    success_url = reverse_lazy("experiments:experiment-repo-list")
+
+    def form_valid(self, form): 
+        response = super().form_valid(form)
+        self.object.clone()
+        return response
 
 # Experiment Views
 def experiment_instances_from_latest(experiment_repos):
@@ -38,12 +53,20 @@ class ExperimentRepoList(ListView):
 
 class ExperimentRepoDetail(DetailView):
     model = models.ExperimentRepo
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        batteries = models.Battery.objects.filter(batteryexperiments__experiment_instance__experiment_repo_id=self.get_object())
+        results = models.Result.objects.filter(battery_experiment__experiment_instance__experiment_repo_id=self.get_object())
+        batt_results = [(batt, list(results.filter(battery_experiment__battery=batt))) for batt in batteries]
+        context['batt_results'] = batt_results
+        return context
 
 
 def add_new_experiments(request):
     created_repos, created_experiments, errors = find_new_experiments()
     for repo in created_repos:
-        messages.info(request, f"Tracking previously unseen repository {repo.origin}")
+        messages.info(request, f"Tracking previously unseen repository {repo.url}")
     for experiment in created_experiments:
         messages.info(request, f"Added new experiment {experiment.name}")
     for error in errors:
@@ -84,7 +107,7 @@ def get_context_data(self, **kwargs):
 
 class BatteryDetail(DetailView):
     model = models.Battery
-    queryset = models.Battery.objects.prefetch_related('experiment_instances')
+    queryset = models.Battery.objects.prefetch_related("assignment_set", "experiment_instances")
 
 
 """
@@ -279,6 +302,8 @@ def post(self, request, *args, **kwargs):
 
 class SubjectList(ListView):
     model = models.Subject
+    queryset = models.Subject.objects.prefetch_related("assignment_set")
+
 
 class CreateSubjects(FormView):
     template_name = 'experiments/create_subjects.html'

expfactory_deploy/templates/base.html

@@ -23,14 +23,15 @@
 
 
     <!-- Your stuff: Third-party CSS libraries go here -->
+    <link href="https://cdn.jsdelivr.net/npm/simple-datatables@latest/dist/style.css" rel="stylesheet" type="text/css">
     {% compress css %}
     <!-- This file stores project-specific CSS -->
 
     <link href="{% static 'css/project.css' %}" rel="stylesheet">
 
     {% endcompress %}
     {% endblock %}
-    <!-- Le javascript
+    <!-- javascript
     ================================================== -->
     {# Placed at the top of the document so pages load faster with defer #}
     {% block javascript %}
@@ -42,7 +43,7 @@
 
       <!-- Your stuff: Third-party javascript libraries go here -->
 
-
+      <script defer src="https://cdn.jsdelivr.net/npm/simple-datatables@latest" type="text/javascript"></script>
       <!-- place project specific Javascript in this file -->
       {% compress js %}
       <script src="{% static 'js/project.js' %}"></script>
@@ -70,25 +71,9 @@
             <li class="nav-item">
               <a class="nav-link" href="{% url 'experiments:experiment-repo-list' %}">Experiments</a>
             </li>
-            {% if request.user.is_authenticated %}
-              <li class="nav-item">
-                {# URL provided by django-allauth/account/urls.py #}
-                <a class="nav-link" href="{% url 'users:detail' request.user.username  %}">{% trans "My Profile" %}</a>
-              </li>
-              <li class="nav-item">
-                {# URL provided by django-allauth/account/urls.py #}
-                <a class="nav-link" href="{% url 'account_logout' %}">{% trans "Sign Out" %}</a>
-              </li>
-            {% else %}
-              <li class="nav-item">
-                {# URL provided by django-allauth/account/urls.py #}
-                <a id="sign-up-link" class="nav-link" href="{% url 'account_signup' %}">{% trans "Sign Up" %}</a>
-              </li>
-              <li class="nav-item">
-                {# URL provided by django-allauth/account/urls.py #}
-                <a id="log-in-link" class="nav-link" href="{% url 'account_login' %}">{% trans "Sign In" %}</a>
-              </li>
-            {% endif %}
+            <li class="nav-item">
+              <a class="nav-link" href="{% url 'experiments:subject-list' %}">Subjects</a>
+            </li>
           </ul>
         </div>
       </nav>