Merge pull request #363 from mp-access/dev

Release v0.10.1

.gitignore

@@ -1,3 +1,10 @@
+# Eclipse-specific files
+.classpath
+.project
+.settings/**/*
+bin/**/*
+
+
 # Compiled class file
 *.class
 
@@ -164,4 +171,5 @@ course_repositories/
 runner/
 courses_db
 
-#load_testing/
+#load_testing/
+

src/main/java/ch/uzh/ifi/access/ServerInfoController.java

@@ -0,0 +1,46 @@
+package ch.uzh.ifi.access;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Component;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.HashMap;
+import java.util.Map;
+
+@RestController
+public class ServerInfoController {
+
+    private final ServerInfo serverInfo;
+
+    public ServerInfoController(ServerInfo serverInfo) {
+        this.serverInfo = serverInfo;
+    }
+
+    @GetMapping("/info")
+    public ResponseEntity<?> info() {
+        Map<String, String> response = new HashMap<>();
+        response.put("offsetDateTime", ZonedDateTime.now().toOffsetDateTime().toString());
+        response.put("utcTime", Instant.now().toString());
+        response.put("zoneId", ZoneId.systemDefault().toString());
+
+        if (serverInfo != null) {
+            response.put("version", serverInfo.version);
+        }
+        return ResponseEntity.ok(response);
+    }
+
+    @Component
+    @Data
+    @Configuration
+    @ConfigurationProperties(prefix = "server.info")
+    static class ServerInfo {
+        private String version;
+    }
+}

src/main/java/ch/uzh/ifi/access/coderunner/CodeRunner.java

@@ -113,7 +113,7 @@ private RunResult createAndRunContainer(ContainerConfig containerConfig, String
 
         ContainerCreation creation = docker.createContainer(containerConfig);
         String containerId = creation.id();
-        logger.trace(String.format("Created container %s", containerId));
+        logger.debug("Created container {}", containerId);
 
         if (creation.warnings() != null) {
             creation.warnings().forEach(logger::warn);
@@ -148,6 +148,8 @@ private RunResult createAndRunContainer(ContainerConfig containerConfig, String
 
         stopAndRemoveContainer(containerId);
 
+        logger.trace("Code execution logs start --------------------\n{}\n-------------------- Code execution logs end", console);
+
         return new RunResult(console, stdOut, stdErr, executionTime, didTimeout, isOomKilled);
     }
 
@@ -184,7 +186,7 @@ private void copyDirectoryToContainer(String containerId, Path folder) throws In
         } catch (IOException e) {
             logger.warn(e.getMessage(), e);
         }
-        logger.trace(joiner.toString());
+        logger.debug(joiner.toString());
     }
 
     private void startAndWaitContainer(String id) throws DockerException, InterruptedException {
@@ -205,7 +207,7 @@ private String readStdErr(String containerId) throws DockerException, Interrupte
     }
 
     private void stopAndRemoveContainer(String id) throws DockerException, InterruptedException {
-        logger.debug(String.format("Stopping and removing container %s", id));
+        logger.debug("Stopping and removing container {}", id);
         docker.stopContainer(id, 1);
         docker.removeContainer(id);
     }

src/main/java/ch/uzh/ifi/access/config/AsyncConfig.java

@@ -42,6 +42,19 @@ public Executor getAsyncExecutor() {
         executor.setCorePoolSize(THREAD_POOL_SIZE);
         executor.setMaxPoolSize(MAX_POOL_SIZE);
         executor.setQueueCapacity(QUEUE_CAPACITY);
+        executor.setWaitForTasksToCompleteOnShutdown(true);
+        executor.setAwaitTerminationSeconds(60);
+        executor.initialize();
+        return executor;
+    }
+
+    @Bean("courseUpdateWorkerExecutor")
+    public Executor getCourseUpdateExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setThreadNamePrefix("course-update-worker-");
+        executor.setCorePoolSize(THREAD_POOL_SIZE);
+        executor.setMaxPoolSize(MAX_POOL_SIZE);
+        executor.setQueueCapacity(QUEUE_CAPACITY);
         executor.initialize();
         return executor;
     }

src/main/java/ch/uzh/ifi/access/config/GracefulShutdown.java

@@ -0,0 +1,25 @@
+package ch.uzh.ifi.access.config;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.event.ContextClosedEvent;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+@Component
+public class GracefulShutdown {
+
+    private static final Logger logger = LoggerFactory.getLogger(GracefulShutdown.class);
+
+    private boolean isShutdown = false;
+
+    @EventListener(ContextClosedEvent.class)
+    public void rejectSubmissionsOnShutdown() {
+        logger.warn("Received shutdown signal. Will reject new submissions");
+        this.isShutdown = true;
+    }
+
+    public boolean isShutdown() {
+        return isShutdown;
+    }
+}

src/main/java/ch/uzh/ifi/access/config/SecurityConfigurer.java

@@ -45,7 +45,7 @@ public void configure(ResourceServerSecurityConfigurer resources) {
 
     @Override
     public void configure(final HttpSecurity http) throws Exception {
-        final String[] swaggerPaths = new String[]{"/v2/api-docs", "/configuration/ui", "/swagger-resources/**", "/configuration/**", "/swagger-ui.html", "/webjars/**"};
+        final String[] permittedPaths = new String[]{"/info", "/v2/api-docs", "/configuration/ui", "/swagger-resources/**", "/configuration/**", "/swagger-ui.html", "/webjars/**"};
 
         http.cors()
                 .configurationSource(corsConfigurationSource())
@@ -58,7 +58,7 @@ public void configure(final HttpSecurity http) throws Exception {
                 .disable()
                 .addFilterAfter(filter, AbstractPreAuthenticatedProcessingFilter.class)
                 .authorizeRequests()
-                .antMatchers(swaggerPaths)
+                .antMatchers(permittedPaths)
                 .permitAll()
                 .antMatchers(securityProperties.getApiMatcher())
                 .authenticated();

src/main/java/ch/uzh/ifi/access/course/controller/CourseController.java

@@ -1,6 +1,5 @@
 package ch.uzh.ifi.access.course.controller;
 
-import ch.uzh.ifi.access.config.ApiTokenAuthenticationProvider;
 import ch.uzh.ifi.access.course.CheckCoursePermission;
 import ch.uzh.ifi.access.course.FilterByPublishingDate;
 import ch.uzh.ifi.access.course.config.CourseAuthentication;
@@ -14,8 +13,10 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.security.authentication.BadCredentialsException;
-import org.springframework.web.bind.annotation.*;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
 import springfox.documentation.annotations.ApiIgnore;
 
 import java.util.ArrayList;
@@ -87,18 +88,4 @@ public ResponseEntity<?> getCourseAssistants(@PathVariable String courseId) {
         UserService.UserQueryResult users = userService.getCourseAdmins(course);
         return ResponseEntity.ok(users.getUsersFound());
     }
-
-    @PostMapping(path = "{id}/update")
-    public void updateCourse(@PathVariable("id") String id, @RequestBody String json,
-                             ApiTokenAuthenticationProvider.GithubHeaderAuthentication authentication) {
-        logger.debug("Received web hook");
-
-        if (!authentication.matchesHmacSignature(json)) {
-            throw new BadCredentialsException("Hmac signature does not match!");
-        }
-
-        logger.debug("Updating courses");
-        courseService.updateCourseById(id);
-    }
-
 }

src/main/java/ch/uzh/ifi/access/course/controller/ExerciseController.java

@@ -15,7 +15,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.util.Map;
 import java.util.Optional;
 
@@ -64,7 +63,7 @@ public ResponseEntity<Resource> getFile(
                 File fileHandle = file.get().getFile();
                 FileSystemResource r = new FileSystemResource(fileHandle);
                 return ResponseEntity.ok()
-                        .contentType(MediaType.parseMediaType(Files.probeContentType(fileHandle.toPath())))
+                        .contentType(MediaType.APPLICATION_OCTET_STREAM)
                         .body(r);
             }
         }
@@ -94,7 +93,7 @@ public ResponseEntity<Resource> searchForFile(
                 File fileHandle = file.get().getFile();
                 FileSystemResource r = new FileSystemResource(fileHandle);
                 return ResponseEntity.ok()
-                        .contentType(MediaType.parseMediaType(Files.probeContentType(fileHandle.toPath())))
+                        .contentType(MediaType.APPLICATION_OCTET_STREAM)
                         .body(r);
             }
         }

src/main/java/ch/uzh/ifi/access/course/controller/WebhooksController.java

@@ -1,12 +1,18 @@
 package ch.uzh.ifi.access.course.controller;
 
 import ch.uzh.ifi.access.config.ApiTokenAuthenticationProvider;
+import ch.uzh.ifi.access.course.model.Course;
 import ch.uzh.ifi.access.course.service.CourseService;
+import com.fasterxml.jackson.databind.JsonNode;
+import lombok.Value;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.http.ResponseEntity;
 import org.springframework.security.authentication.BadCredentialsException;
 import org.springframework.web.bind.annotation.*;
 
+import java.util.Optional;
+
 @RestController
 @RequestMapping("/webhooks")
 public class WebhooksController {
@@ -32,6 +38,17 @@ public void updateCourse(@PathVariable("id") String id, @RequestBody String json
         courseService.updateCourseById(id);
     }
 
+    @PostMapping(path = "/courses/update/github")
+    public ResponseEntity<?> updateCourse(@RequestBody JsonNode payload, ApiTokenAuthenticationProvider.GithubHeaderAuthentication authentication) {
+        logger.info("Received github web hook");
+
+        if (!authentication.matchesHmacSignature(payload.toString())) {
+            throw new BadCredentialsException("Hmac signature does not match!");
+        }
+
+        return processWebhook(payload, false);
+    }
+
     @PostMapping(path = "/courses/{id}/update/gitlab")
     public void updateCourse(@PathVariable("id") String id,
                              ApiTokenAuthenticationProvider.GitlabHeaderAuthentication authentication) {
@@ -44,4 +61,61 @@ public void updateCourse(@PathVariable("id") String id,
         logger.info("Updating courses");
         courseService.updateCourseById(id);
     }
+
+    @PostMapping(path = "/courses/update/gitlab")
+    public ResponseEntity<?> updateCourse(@RequestBody JsonNode payload, ApiTokenAuthenticationProvider.GitlabHeaderAuthentication authentication) {
+        logger.info("Received gitlab web hook");
+
+        if (!authentication.isMatchesSecret()) {
+            throw new BadCredentialsException("Header secret does not match!");
+        }
+
+        return processWebhook(payload, true);
+    }
+
+    private ResponseEntity<String> processWebhook(JsonNode payload, boolean isGitlab) {
+        logger.info("Updating course");
+        WebhookPayload webhookPayload = new WebhookPayload(payload, isGitlab);
+        Optional<Course> courseToUpdate = courseService.getAllCourses().stream().filter(course -> webhookPayload.matchesCourseUrl(course.getGitURL())).findFirst();
+        courseToUpdate.ifPresent(c -> courseService.updateCourseById(c.getId()));
+        return courseToUpdate.map(c -> ResponseEntity.accepted().body(c.getId())).orElse(ResponseEntity.notFound().build());
+    }
+
+    @Value
+    public static class WebhookPayload {
+
+        private JsonNode repository;
+
+        private boolean isGitlab;
+
+        public WebhookPayload(JsonNode root, boolean isGitlab) {
+            this.repository = root.get("repository");
+            this.isGitlab = isGitlab;
+        }
+
+        public String getHtmlUrl() {
+            if (isGitlab) {
+                return repository.get("homepage").asText();
+            }
+            return repository.get("html_url").asText();
+        }
+
+        public String getGitUrl() {
+            if (isGitlab) {
+                return repository.get("git_http_url").asText();
+            }
+            return repository.get("clone_url").asText();
+        }
+
+        public String getSshUrl() {
+            if (isGitlab) {
+                return repository.get("git_ssh_url").asText();
+            }
+            return repository.get("ssh_url").asText();
+        }
+
+        public boolean matchesCourseUrl(String courseUrl) {
+            return courseUrl.equalsIgnoreCase(getHtmlUrl()) || courseUrl.equalsIgnoreCase(getGitUrl()) || courseUrl.equalsIgnoreCase(getSshUrl());
+        }
+    }
 }

src/main/java/ch/uzh/ifi/access/course/dao/CourseDAO.java

@@ -10,6 +10,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
 import lombok.Data;
+import org.apache.commons.lang.SerializationUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.core.io.ClassPathResource;
@@ -40,15 +41,18 @@ public class CourseDAO {
 
     private BreakingChangeNotifier breakingChangeNotifier;
 
-    public CourseDAO(BreakingChangeNotifier breakingChangeNotifier) {
+    private RepoCacher repoCacher;
+
+    public CourseDAO(BreakingChangeNotifier breakingChangeNotifier, RepoCacher repoCacher) {
         this.breakingChangeNotifier = breakingChangeNotifier;
+        this.repoCacher = repoCacher;
 
         ClassPathResource resource = new ClassPathResource(CONFIG_FILE);
         if (resource.exists()) {
             try {
                 ObjectMapper mapper = new ObjectMapper();
                 URLList conf = mapper.readValue(resource.getFile(), URLList.class);
-                courseList = RepoCacher.retrieveCourseData(conf.repositories);
+                courseList = repoCacher.retrieveCourseData(conf.repositories);
                 exerciseIndex = buildExerciseIndex(courseList);
                 logger.info(String.format("Parsed %d courses", courseList.size()));
 
@@ -95,14 +99,35 @@ protected Map<String, Exercise> buildExerciseIndex(List<Course> courses) {
     public Course updateCourseById(String id) {
         Course c = selectCourseById(id)
                 .orElseThrow(() -> new ResourceNotFoundException("No course found"));
+
+        logger.info("Updating course {} {}", c.getTitle(), id);
+
+        return updateCourse(c);
+    }
+
+    protected Course updateCourse(Course c) {
+        Course clone = (Course) SerializationUtils.clone(c);
+        Course newCourse;
+
+        // Try to pull new course
         try {
-            Course courseUpdate = RepoCacher.retrieveCourseData(new String[]{c.getGitURL()}).get(0);
-            updateCourse(c, courseUpdate);
-            return c;
+            newCourse = repoCacher.retrieveCourseData(new String[]{c.getGitURL()}).get(0);
         } catch (Exception e) {
+            logger.error("Failed to generate new course", e);
+            return null;
+        }
+
+        // Try to update
+        try {
+            updateCourse(c, newCourse);
+        } catch (Exception e) {
+            // If we fail during updating we try to revert to original
             logger.error("Failed to update course", e);
+            updateCourse(c, clone);
+            return null;
         }
-        return null;
+
+        return c;
     }
 
     void updateCourse(Course before, Course after) {