From a7fd032d519d374f1e21d5f67fba7de2fd01a213 Mon Sep 17 00:00:00 2001 From: Lai Wei Date: Mon, 7 Oct 2024 12:24:17 +0100 Subject: [PATCH] Improve how scheduled tasks handle missing resources --- .../o365/classes/feature/cohortsync/main.php | 66 +-- .../o365/classes/feature/coursesync/main.php | 482 ++++++++++++++---- local/o365/classes/task/cohortsync.php | 26 +- local/o365/classes/task/coursesync.php | 11 + local/o365/classes/utils.php | 132 +++++ local/o365/db/install.xml | 3 +- local/o365/db/upgrade.php | 17 +- local/o365/version.php | 2 +- 8 files changed, 562 insertions(+), 177 deletions(-) diff --git a/local/o365/classes/feature/cohortsync/main.php b/local/o365/classes/feature/cohortsync/main.php index 8418ea6b3..6471e081f 100644 --- a/local/o365/classes/feature/cohortsync/main.php +++ b/local/o365/classes/feature/cohortsync/main.php @@ -42,11 +42,6 @@ * Microsoft Group and Moodle cohort mapping feature main class. */ class main { - /** - * API error message for group not existing. - */ - const GROUP_DOES_NOT_EXIST_ERROR = "does not exist or one of its queried reference-property objects are not present"; - /** * @var unified Graph client instance. */ @@ -185,7 +180,10 @@ public function delete_mapping_by_id(int $id) : void { public function fetch_groups_from_cache() : void { global $DB; - $records = $DB->get_records('local_o365_groups_cache'); + $sql = 'SELECT * + FROM {local_o365_groups_cache} + WHERE not_found_since = 0'; + $records = $DB->get_records_sql($sql); $this->grouplist = []; foreach ($records as $record) { @@ -213,56 +211,16 @@ public function fetch_cohorts() : void { public function update_groups_cache() : bool { global $DB; - mtrace("... Start updating groups cache."); - - try { - $this->fetch_groups_from_microsoft(); - } catch (moodle_exception $e) { - mtrace("...... Failed to fetch groups. Error: " . $e->getMessage()); + if (utils::update_groups_cache($this->graphclient, 1)) { + $sql = 'SELECT * + FROM {local_o365_groups_cache} + WHERE not_found_since = 0'; + $this->grouplist = $DB->get_records_sql($sql); + return true; + } else { return false; } - - $existingcacherecords = $DB->get_records('local_o365_groups_cache'); - $existingcachebyoid = []; - foreach ($existingcacherecords as $existingcacherecord) { - $existingcachebyoid[$existingcacherecord->objectid] = $existingcacherecord; - } - - foreach ($this->grouplist as $group) { - if (array_key_exists($group['id'], $existingcachebyoid)) { - $cacherecord = $existingcachebyoid[$group['id']]; - $cacherecord->name = $group['displayName']; - $cacherecord->description = $group['description']; - $DB->update_record('local_o365_groups_cache', $cacherecord); - unset($existingcachebyoid[$group['id']]); - } else { - $cacherecord = new stdClass(); - $cacherecord->objectid = $group['id']; - $cacherecord->name = $group['displayName']; - $cacherecord->description = $group['description']; - $DB->insert_record('local_o365_groups_cache', $cacherecord); - mtrace("...... Added group ID {$group['id']} to cache."); - } - } - - foreach ($existingcachebyoid as $oldcacherecord) { - $DB->delete_records('local_o365_groups_cache', ['id' => $oldcacherecord->id]); - mtrace("...... Deleted group ID {$oldcacherecord->objectid} from cache."); - } - - mtrace("... Finished updating groups cache."); - - return true; - } - - /** - * Fetch all Microsoft groups. - * - * @return void - */ - public function fetch_groups_from_microsoft() : void { - $this->grouplist = $this->graphclient->get_groups(); } /** @@ -299,7 +257,7 @@ public function get_group_owners_and_members(string $groupoid) { $memberrecords = $this->graphclient->get_group_members($groupoid); $ownerrecords = $this->graphclient->get_group_owners($groupoid); } catch (moodle_exception $e) { - if (strpos($e->getMessage(), self::GROUP_DOES_NOT_EXIST_ERROR) !== false) { + if (strpos($e->getMessage(), utils::RESOURCE_NOT_EXIST_ERROR) !== false) { $DB->delete_records('local_o365_objects', ['objectid' => $groupoid]); mtrace("...... Deleted mapping for non-existing group ID $groupoid."); } else { diff --git a/local/o365/classes/feature/coursesync/main.php b/local/o365/classes/feature/coursesync/main.php index 4596a732f..618879824 100644 --- a/local/o365/classes/feature/coursesync/main.php +++ b/local/o365/classes/feature/coursesync/main.php @@ -27,6 +27,7 @@ namespace local_o365\feature\coursesync; use context_course; +use core\lock\lock_config; use core\task\manager; use local_o365\rest\unified; use moodle_exception; @@ -101,9 +102,7 @@ public function __construct(unified $graphclient, bool $debug = false) { */ protected function mtrace(string $msg, int $level = 0, string $eol = "\n") { if ($this->debug === true) { - if ($level) { - $msg = str_repeat('...', $level) . ' ' . $msg; - } + $msg = str_repeat('...', $level + 1) . ' ' . $msg; mtrace($msg, $eol); } } @@ -124,18 +123,20 @@ public function sync_courses() : bool { global $DB; $this->mtrace('Start syncing courses.'); - $this->mtrace('Tenant has education license: ' . ($this->haseducationlicense ? 'yes' : 'no')); + + $this->mtrace('Tenant has education license: ' . ($this->haseducationlicense ? 'yes' : 'no'), 1); + $this->mtrace('', 1); // Preparation work - get list of courses that have course sync enabled. $coursesyncsetting = get_config('local_o365', 'coursesync'); if ($coursesyncsetting === 'onall' || $coursesyncsetting === 'oncustom') { $coursesenabled = utils::get_enabled_courses(); if (empty($coursesenabled)) { - $this->mtrace('Custom group creation is enabled, but no courses are enabled.'); + $this->mtrace('Custom course sync is enabled, but no courses are enabled.'); return false; } } else { - $this->mtrace('Group creation is disabled.'); + $this->mtrace('Course sync is disabled.'); return false; } @@ -147,24 +148,29 @@ public function sync_courses() : bool { } // Process courses with groups that have been "soft-deleted". - $this->restore_soft_deleted_groups(); + $this->restore_soft_deleted_groups(1); // Process courses without an associated group. - $this->process_courses_without_groups(); + $this->process_courses_without_groups(1); // Process courses having groups but not teams. $this->process_courses_without_teams(); + $this->mtrace('Finished syncing courses.'); + $this->mtrace(''); + return true; } /** * Restore Microsoft 365 groups that have been soft-deleted. + * + * @param int $baselevel */ - private function restore_soft_deleted_groups() { + private function restore_soft_deleted_groups($baselevel = 1) { global $DB; - $this->mtrace('Restore groups that have been soft-deleted...'); + $this->mtrace('Restore groups that have been soft-deleted...', $baselevel); $sql = 'SELECT crs.id as courseid, obj.* FROM {course} crs @@ -179,30 +185,34 @@ private function restore_soft_deleted_groups() { foreach ($objectrecs as $objectrec) { $metadata = (!empty($objectrec->metadata)) ? @json_decode($objectrec->metadata, true) : []; if (is_array($metadata) && !empty($metadata['softdelete'])) { - $this->mtrace('Attempting to restore group for course #' . $objectrec->courseid, 1); + $this->mtrace('Attempting to restore group for course #' . $objectrec->courseid, $baselevel + 1); $result = $this->restore_group($objectrec->id, $objectrec->objectid, $metadata); if ($result === true) { - $this->mtrace('success!', 2); + $this->mtrace('success!', $baselevel + 2); } else { - $this->mtrace('failed. Group may have been deleted for too long.', 2); + $this->mtrace('failed. Group may have been deleted for too long.', $baselevel + 2); // TODO do we need to delete group record in object table then? } } } + + $this->mtrace('Finished restoring groups that have been soft-deleted.', $baselevel); + $this->mtrace('', $baselevel); } /** * Create an educationClass group for the given course. * * @param stdClass $course + * @param int $baselevel * @return array|false */ - private function create_education_group(stdClass $course) { + private function create_education_group(stdClass $course, int $baselevel = 3) { global $DB; $now = time(); - $this->mtrace('Creating education group for course #' . $course->id, 2); + $this->mtrace('Create education group for course #' . $course->id, $baselevel); $displayname = utils::get_team_display_name($course); $mailnickname = utils::get_group_mail_alias($course); @@ -220,17 +230,18 @@ private function create_education_group(stdClass $course) { $response = $this->graphclient->create_educationclass_group($displayname, $mailnickname, $description, $externalid, $externalname); } catch (moodle_exception $e) { - $this->mtrace('Could not create educationClass group for course #' . $course->id . '. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Could not create educationClass group for course #' . $course->id . '. Reason: ' . $e->getMessage(), + $baselevel + 1); return false; } - $this->mtrace('Created education group ' . $response['id'] . ' for course #' . $course->id, 3); + $this->mtrace('Created education group ' . $response['id'] . ' for course #' . $course->id, $baselevel + 1); $objectrecord = ['type' => 'group', 'subtype' => 'course', 'objectid' => $response['id'], 'moodleid' => $course->id, 'o365name' => $displayname, 'timecreated' => $now, 'timemodified' => $now]; $objectrecord['id'] = $DB->insert_record('local_o365_objects', (object)$objectrecord); $this->mtrace('Recorded group object ' . $objectrecord['objectid'] . ' into object table with record ID ' . - $objectrecord['id'], 3); + $objectrecord['id'], $baselevel + 1); return $objectrecord; } @@ -240,10 +251,11 @@ private function create_education_group(stdClass $course) { * * @param string $groupobjectid * @param stdClass $course + * @param int $baselevel * @return bool */ - public function set_lti_properties_in_education_group(string $groupobjectid, stdClass $course) : bool { - $this->mtrace('Setting LMS attributes in group ' . $groupobjectid . ' for course #' . $course->id, 2); + public function set_lti_properties_in_education_group(string $groupobjectid, stdClass $course, int $baselevel = 3) : bool { + $this->mtrace('Set LMS attributes in group ' . $groupobjectid . ' for course #' . $course->id, $baselevel); $lmsattributes = [ 'microsoft_EducationClassLmsExt' => [ @@ -257,7 +269,7 @@ public function set_lti_properties_in_education_group(string $groupobjectid, std $success = false; while ($retrycounter <= API_CALL_RETRY_LIMIT) { if ($retrycounter) { - $this->mtrace('Retry #' . $retrycounter, 3); + $this->mtrace('Retry #' . $retrycounter, $baselevel + 1); } sleep(10); @@ -266,15 +278,16 @@ public function set_lti_properties_in_education_group(string $groupobjectid, std $success = true; break; } catch (moodle_exception $e) { - $this->mtrace('Error setting LMS attributes in group ' . $groupobjectid . '. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Error setting LMS attributes in group ' . $groupobjectid . '. Reason: ' . $e->getMessage(), + $baselevel + 1); $retrycounter++; } } if ($success) { - $this->mtrace('Successfully setting LMS attributes.', 3); + $this->mtrace('Successfully setting LMS attributes.', $baselevel + 1); } else { - $this->mtrace('Failed setting LMS attributes.', 3); + $this->mtrace('Failed setting LMS attributes.', $baselevel + 1); } return $success; @@ -284,14 +297,15 @@ public function set_lti_properties_in_education_group(string $groupobjectid, std * Create a standard group for the given course. * * @param stdClass $course + * @param int $baselevel * @return array|false */ - private function create_standard_group(stdClass $course) { + private function create_standard_group(stdClass $course, int $baselevel = 3) { global $DB; $now = time(); - $this->mtrace('Creating standard group for course #' . $course->id, 2); + $this->mtrace('Create standard group for course #' . $course->id, $baselevel); $displayname = utils::get_team_display_name($course); $mailnickname = utils::get_group_mail_alias($course); @@ -306,17 +320,18 @@ private function create_standard_group(stdClass $course) { try { $response = $this->graphclient->create_group($displayname, $mailnickname, ['description' => $description]); } catch (moodle_exception $e) { - $this->mtrace('Could not create standard group for course #' . $course->id . '. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Could not create standard group for course #' . $course->id . '. Reason: ' . $e->getMessage(), + $baselevel + 1); return false; } - $this->mtrace('Created standard group ' . $response['id'] . ' for course #' . $course->id, 3); + $this->mtrace('Created standard group ' . $response['id'] . ' for course #' . $course->id, $baselevel); $objectrecord = ['type' => 'group', 'subtype' => 'course', 'objectid' => $response['id'], 'moodleid' => $course->id, 'o365name' => $displayname, 'timecreated' => $now, 'timemodified' => $now]; $objectrecord['id'] = $DB->insert_record('local_o365_objects', (object)$objectrecord); $this->mtrace('Recorded group object (' . $objectrecord['objectid'] . ') into object table with record ID ' . - $objectrecord['id'], 3); + $objectrecord['id'], $baselevel); return $objectrecord; } @@ -327,107 +342,239 @@ private function create_standard_group(stdClass $course) { * @param string $groupobjectid * @param array $owners * @param array $members + * @param int $baselevel * @return bool whether at least one owner was added. */ - private function add_group_owners_and_members_to_group(string $groupobjectid, array $owners, array $members) : bool { + private function add_group_owners_and_members_to_group(string $groupobjectid, array $owners, array $members, + int $baselevel = 3) : bool { + global $SESSION; if (empty($owners) && empty($members)) { - $this->mtrace('Skip adding owners / members to the group. Reason: No users to add.', 2); + $this->mtrace('Skip adding owners / members to the group. Reason: No users to add.', $baselevel); return false; } // Remove existing owners and members. try { - $existingowners = $this->get_group_owners($groupobjectid); + $skip = false; + $existingowners = []; + + $this->mtrace('Get existing owners of group with ID ' . $groupobjectid, $baselevel); + + if (isset($SESSION->o365_groups_not_exist)) { + if (in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + $skip = true; + } + } + if (!$skip) { + $existingowners = $this->get_group_owners($groupobjectid); + } } catch (moodle_exception $e) { - $this->mtrace('Could not get existing owners of group with ID ' . $groupobjectid . '. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Could not get existing owners of group with ID ' . $groupobjectid . '. Reason: ' . $e->getMessage(), + $baselevel + 1); $existingowners = []; + + if (isset($SESSION->o365_groups_not_exist) && isset($SESSION->o365_newly_created_groups)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + if (stripos($e->getMessage(), $groupobjectid) !== false) { + // The group doesn't exist. + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $SESSION->o365_groups_not_exist[] = $groupobjectid; + } + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + } + } + } } try { - $existingmembers = $this->get_group_members($groupobjectid); + $skip = false; + $existingmembers = []; + + $this->mtrace('Get existing members of group with ID ' . $groupobjectid, $baselevel); + + if (isset($SESSION->o365_groups_not_exist)) { + if (in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + $skip = true; + } + } + if (!$skip) { + $existingmembers = $this->get_group_members($groupobjectid); + } } catch (moodle_exception $e) { - $this->mtrace('Could not get existing members of group with ID ' . $groupobjectid . '. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Could not get existing members of group with ID ' . $groupobjectid . '. Reason: ' . $e->getMessage(), + $baselevel + 1); $existingmembers = []; + + if (isset($SESSION->o365_groups_not_exist)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + if (stripos($e->getMessage(), $groupobjectid) !== false) { + // The group doesn't exist. + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $SESSION->o365_groups_not_exist[] = $groupobjectid; + } + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + } + } + } } $existingownerids = array_keys($existingowners); $existingmemberids = array_keys($existingmembers); $owners = array_diff($owners, $existingownerids); $members = array_diff($members, $existingmemberids); - $this->mtrace('Adding ' . count($owners) . ' owners and ' . count($members) . ' members to group with ID ' . - $groupobjectid, 2); + $this->mtrace('Add ' . count($owners) . ' owners and ' . count($members) . ' members to group with ID ' . + $groupobjectid, $baselevel); $userchunks = utils::arrange_group_users_in_chunks($owners, $members); $owneradded = false; - foreach ($userchunks as $userchunk) { + + foreach ($userchunks as $key => $userchunk) { $role = array_keys($userchunk)[0]; $users = reset($userchunk); $retrycounter = 0; while ($retrycounter <= API_CALL_RETRY_LIMIT) { if ($retrycounter) { - $this->mtrace('Retry #' . $retrycounter, 3); + $this->mtrace('Retry #' . $retrycounter, $baselevel + 1); sleep(10); } try { + $this->mtrace('Chunk ' . $key + 1 . ', adding ' . count($users) . ' users as ' . $role, $baselevel + 1); + + if (isset($SESSION->o365_groups_not_exist)) { + if (in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $this->mtrace('Group does not exist. Skipping.', $baselevel + 2); + break; + } + } + $response = $this->graphclient->add_chunk_users_to_group($groupobjectid, $role, $users); if ($response) { if ($role == 'owner') { $owneradded = true; } } else { - $this->mtrace('Invalid bulk group owners/members addition request', 3); + $this->mtrace('Invalid bulk group owners/members addition request', $baselevel + 2); } break; } catch (moodle_exception $e) { - $this->mtrace('Error: ' . $e->getMessage(), 3); + $this->mtrace('Error: ' . $e->getMessage(), $baselevel + 2); + if (isset($SESSION->o365_groups_not_exist) && isset($SESSION->o365_newly_created_groups) && + isset($SESSION->o365_users_not_exist)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + if (stripos($e->getMessage(), $groupobjectid) !== false) { + // The non-existing resource is the group. + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $SESSION->o365_groups_not_exist[] = $groupobjectid; + } + $this->mtrace('Group does not exist. Skip retries.', $baselevel + 2); + break; + } else { + // The non-existing resource is a user. + $useroid = \local_o365\utils::extract_guid_from_error_message($e->getMessage()); + if (!empty($useroid) && !in_array($useroid, $SESSION->o365_users_not_exist)) { + $SESSION->o365_users_not_exist[] = $useroid; + $this->mtrace('User ' . $useroid . ' does not exist. Skip retries.', $baselevel + 2); + } else { + $this->mtrace('User does not exist. Skip retries.', $baselevel + 2); + } + break; + } + } + } $retrycounter++; } } } - $this->mtrace('Finished adding owners and members to group.', 3); - return $owneradded; } + /** + * Check if the exception message is about a resource not existing. + * + * @param string $exceptionmessage + * @return bool + */ + private static function is_resource_not_exist_exception(string $exceptionmessage) : bool { + return (strpos($exceptionmessage, \local_o365\utils::RESOURCE_NOT_EXIST_ERROR) !== false); + } + /** * Create a class team from an education group with the given ID for the course with the given ID. * * @param string $groupobjectid * @param stdClass $course + * @param int $baselevel * @return array|false */ - private function create_class_team_from_education_group(string $groupobjectid, stdClass $course) { - global $DB; + private function create_class_team_from_education_group(string $groupobjectid, stdClass $course, int $baselevel = 3) { + global $DB, $SESSION; $now = time(); $retrycounter = 0; - $this->mtrace('Creating class team from education group with ID ' . $groupobjectid . ' for course #' . $course->id, 2); + $this->mtrace('Create class team from education group with ID ' . $groupobjectid . ' for course #' . $course->id, + $baselevel); $response = null; $subtype = ''; while ($retrycounter <= API_CALL_RETRY_LIMIT) { + if (isset($SESSION->o365_groups_not_exist)) { + if (in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + break; + } + } + if ($retrycounter) { - $this->mtrace('Retry #' . $retrycounter, 3); + $this->mtrace('Retry #' . $retrycounter, $baselevel + 1); + sleep(10); } - sleep(10); try { $response = $this->graphclient->create_class_team_from_education_group($groupobjectid); - $this->mtrace('Created class team from class group with ID ' . $groupobjectid, 3); + $this->mtrace('Created class team from class group with ID ' . $groupobjectid, $baselevel + 1); $subtype = 'teamfromgroup'; break; } catch (moodle_exception $e) { if (strpos($e->a, 'The group is already provisioned') !== false) { - $this->mtrace('Found existing team from class group with ID ' . $groupobjectid, 3); + $this->mtrace('Found existing team from class group with ID ' . $groupobjectid, $baselevel + 1); $response = true; $subtype = 'courseteam'; break; } else { - $this->mtrace('Could not create class team from education group. Reason: ' . $e->getMessage(), 3); + $this->mtrace('Could not create class team from education group. Reason: ' . $e->getMessage(), $baselevel + 1); + + if (isset($SESSION->o365_groups_not_exist) && isset($SESSION->o365_newly_created_groups) && + isset($SESSION->o365_users_not_exist)) { + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + if (stripos($e->getMessage(), $groupobjectid) !== false) { + // The non-existing resource is the group. + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $SESSION->o365_groups_not_exist[] = $groupobjectid; + } + $this->mtrace('Group does not exist. Skip retries.', $baselevel + 2); + break; + } else { + // The non-existing resource is a user. + $useroid = \local_o365\utils::extract_guid_from_error_message($e->getMessage()); + if (!empty($useroid) && !in_array($useroid, $SESSION->o365_users_not_exist)) { + $SESSION->o365_users_not_exist[] = $useroid; + $this->mtrace('User ' . $useroid . ' does not exist. Skip retries.', $baselevel + 2); + } else { + $this->mtrace('User does not exist. Skip retries.', $baselevel + 2); + } + break; + } + } + } + } + $retrycounter++; } } @@ -435,7 +582,7 @@ private function create_class_team_from_education_group(string $groupobjectid, s if (!$response) { $this->mtrace('Failed to create class team from education group with ID ' . $groupobjectid . ' for course #' . - $course->id, 3); + $course->id, $baselevel + 1); return false; } @@ -444,10 +591,10 @@ private function create_class_team_from_education_group(string $groupobjectid, s 'moodleid' => $course->id, 'o365name' => $teamname, 'timecreated' => $now, 'timemodified' => $now]; $teamobjectrecord['id'] = $DB->insert_record('local_o365_objects', (object)$teamobjectrecord); $this->mtrace('Recorded class team object ' . $groupobjectid . ' into object table with record ID ' . - $teamobjectrecord['id'], 3); + $teamobjectrecord['id'], $baselevel + 1); // Provision app, add app tab to channel. - $this->install_moodle_app_in_team($groupobjectid, $course->id); + $this->install_moodle_app_in_team($groupobjectid, $course->id, $baselevel + 1); return $teamobjectrecord; } @@ -457,47 +604,82 @@ private function create_class_team_from_education_group(string $groupobjectid, s * * @param string $groupobjectid * @param stdClass $course + * @param int $baselevel * @return array|false */ - private function create_team_from_standard_group(string $groupobjectid, stdClass $course) { - global $DB; + private function create_team_from_standard_group(string $groupobjectid, stdClass $course, int $baselevel = 3) { + global $DB, $SESSION; $now = time(); $retrycounter = 0; - $this->mtrace('Creating standard team from group with ID ' . $groupobjectid . ' for course #' . $course->id, 2); + $this->mtrace('Create standard team from group with ID ' . $groupobjectid . ' for course #' . $course->id, $baselevel); $response = null; while ($retrycounter <= API_CALL_RETRY_LIMIT) { if ($retrycounter) { - $this->mtrace('Retry #' . $retrycounter, 3); + $this->mtrace('Retry #' . $retrycounter, $baselevel + 1); sleep(10); } try { + if (isset($SESSION->o365_groups_not_exist)) { + if (in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $this->mtrace('Group does not exist. Skipping.', $baselevel + 1); + break; + } + } $response = $this->graphclient->create_standard_team_from_group($groupobjectid); break; } catch (moodle_exception $e) { - $this->mtrace('Could not create standard team from group. Reason: '. $e->getMessage(), 3); + $this->mtrace('Could not create standard team from group. Reason: '. $e->getMessage(), $baselevel + 1); + + if (isset($SESSION->o365_groups_not_exist) && isset($SESSION->o365_newly_created_groups) && + isset($SESSION->o365_users_not_exist)) { + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + if (stripos($e->getMessage(), $groupobjectid) !== false) { + // The non-existing resource is the group. + if (!in_array($groupobjectid, $SESSION->o365_groups_not_exist)) { + $SESSION->o365_groups_not_exist[] = $groupobjectid; + } + $this->mtrace('Group does not exist. Skip retries.', $baselevel + 2); + break; + } else { + // The non-existing resource is a user. + $useroid = \local_o365\utils::extract_guid_from_error_message($e->getMessage()); + if (!empty($useroid) && !in_array($useroid, $SESSION->o365_users_not_exist)) { + $SESSION->o365_users_not_exist[] = $useroid; + $this->mtrace('User ' . $useroid . ' does not exist. Skip retries.', $baselevel + 2); + } else { + $this->mtrace('User does not exist. Skip retries.', $baselevel + 2); + } + break; + } + } + } + } + $retrycounter++; } } if (!$response) { - $this->mtrace('Failed to create standard team from group with ID ' . $groupobjectid . ' for course #' . $course->id, 3); + $this->mtrace('Failed to create standard team from group with ID ' . $groupobjectid . ' for course #' . $course->id, + $baselevel + 1); return false; } - $this->mtrace('Created standard team from group with ID ' . $groupobjectid, 3); + $this->mtrace('Created standard team from group with ID ' . $groupobjectid, $baselevel + 1); $teamname = utils::get_team_display_name($course); $teamobjectrecord = ['type' => 'group', 'subtype' => 'teamfromgroup', 'objectid' => $groupobjectid, 'moodleid' => $course->id, 'o365name' => $teamname, 'timecreated' => $now, 'timemodified' => $now]; $teamobjectrecord['id'] = $DB->insert_record('local_o365_objects', (object)$teamobjectrecord); $this->mtrace('Recorded standard team object ' . $groupobjectid . ' into object table with record ID ' . - $teamobjectrecord['id'], 3); + $teamobjectrecord['id'], $baselevel + 1); // Provision app, add app tab to channel. - $this->install_moodle_app_in_team($groupobjectid, $course->id); + $this->install_moodle_app_in_team($groupobjectid, $course->id, $baselevel + 1); return $teamobjectrecord; } @@ -507,30 +689,31 @@ private function create_team_from_standard_group(string $groupobjectid, stdClass * * @param string $groupobjectid * @param int $courseid + * @param int $baselevel */ - public function install_moodle_app_in_team(string $groupobjectid, int $courseid) { + public function install_moodle_app_in_team(string $groupobjectid, int $courseid, int $baselevel = 4) { $moodleappid = get_config('local_o365', 'moodle_app_id'); if (!empty($moodleappid)) { // Provision app to the newly created team. - $this->mtrace('Provision Moodle app in the team', 3); + $this->mtrace('Provision Moodle app in the team', $baselevel); $retrycounter = 0; $moodleappprovisioned = false; while ($retrycounter <= API_CALL_RETRY_LIMIT) { if ($retrycounter) { - $this->mtrace('Retry #' . $retrycounter, 4); + $this->mtrace('Retry #' . $retrycounter, $baselevel + 1); } sleep(10); try { if ($this->graphclient->provision_app($groupobjectid, $moodleappid)) { - $this->mtrace('Provisioned Moodle app in the team with object ID ' . $groupobjectid, 4); + $this->mtrace('Provisioned Moodle app in the team with object ID ' . $groupobjectid, $baselevel + 1); $moodleappprovisioned = true; break; } } catch (moodle_exception $e) { $this->mtrace('Could not add app to team with object ID ' . $groupobjectid . '. Reason: ' . $e->getMessage(), - 4); + $baselevel + 1); $retrycounter++; } } @@ -539,21 +722,22 @@ public function install_moodle_app_in_team(string $groupobjectid, int $courseid) if ($moodleappprovisioned) { try { $generalchanelid = $this->graphclient->get_general_channel_id($groupobjectid); - $this->mtrace('Located general channel in the team with object ID ' . $groupobjectid, 4); + $this->mtrace('Located general channel in the team with object ID ' . $groupobjectid, $baselevel + 1); } catch (moodle_exception $e) { $this->mtrace('Could not list channels of team with object ID ' . $groupobjectid . '. Reason: ' . - $e->getMessage(), 4); + $e->getMessage(), $baselevel + 1); $generalchanelid = false; } if ($generalchanelid) { // Add tab to channel. try { - $this->add_moodle_tab_to_channel($groupobjectid, $generalchanelid, $moodleappid, $courseid); - $this->mtrace('Installed Moodle tab in the general channel of team with object ID ' . $groupobjectid, 4); + $this->add_moodle_tab_to_channel($groupobjectid, $generalchanelid, $moodleappid, $courseid, $baselevel + 1); + $this->mtrace('Installed Moodle tab in the general channel of team with object ID ' . $groupobjectid, + $baselevel + 1); } catch (moodle_exception $e) { $this->mtrace('Could not add Moodle tab to channel in team with ID ' . $groupobjectid . '. Reason : ' . - $e->getMessage(), 4); + $e->getMessage(), $baselevel + 1); } } } @@ -587,17 +771,19 @@ private function add_moodle_tab_to_channel(string $groupobjectid, string $channe * - Create groups, * - Add owners and members, * - Create Teams if appropriate. + * + * @param int $baselevel */ - private function process_courses_without_groups() { + private function process_courses_without_groups(int $baselevel = 1) { global $DB; - $this->mtrace('Processing courses without groups...'); + $this->mtrace('Process courses without groups...', $baselevel); // Process adhoc tasks first to prevent creating duplicate teams for the same course. $courserequestadhoctasks = manager::get_adhoc_tasks('local_o365\task\processcourserequestapproval'); foreach ($courserequestadhoctasks as $courserequestadhoctask) { manager::adhoc_task_starting($courserequestadhoctask); - $cronlockfactory = \core\lock\lock_config::get_lock_factory('local_o365'); + $cronlockfactory = lock_config::get_lock_factory('local_o365'); if ($lock = $cronlockfactory->get_lock('\\' . get_class($courserequestadhoctask), 10)) { $courserequestadhoctask->set_lock($lock); $courserequestadhoctask->execute(); @@ -626,20 +812,20 @@ private function process_courses_without_groups() { foreach ($courses as $course) { if ($coursesprocessed > $courselimit) { - $this->mtrace('Course processing limit of ' . $courselimit . ' reached. Exit.'); + $this->mtrace('Course processing limit of ' . $courselimit . ' reached. Exit.', $baselevel); break; } - if ($this->create_group_for_course($course)) { + if ($this->create_group_for_course($course, $baselevel + 1)) { $coursesprocessed++; } } - if (empty($coursesprocessed)) { - $this->mtrace('All courses have groups created.', 1); - } else { - $this->mtrace('Created groups for ' . $coursesprocessed . ' courses.', 1); + $this->mtrace('Finished processing courses without groups.', $baselevel); + if ($coursesprocessed) { + $this->mtrace('Created groups for ' . $coursesprocessed . ' courses.', $baselevel); } + $this->mtrace('', $baselevel); $courses->close(); } @@ -648,42 +834,52 @@ private function process_courses_without_groups() { * Try to create a group for the given course. * * @param stdClass $course + * @param int $baselevel * @return bool True if group creation succeeds, or False if it fails. */ - public function create_group_for_course(stdClass $course) : bool { - $this->mtrace('Processing course #' . $course->id, 1); + public function create_group_for_course(stdClass $course, int $baselevel = 2) : bool { + global $SESSION; + + $this->mtrace('Process course #' . $course->id, $baselevel); // Create group. if ($this->haseducationlicense) { - $groupobject = $this->create_education_group($course); + $groupobject = $this->create_education_group($course, $baselevel + 1); if ($groupobject) { - $this->set_lti_properties_in_education_group($groupobject['objectid'], $course); + $this->set_lti_properties_in_education_group($groupobject['objectid'], $course, $baselevel + 1); } else { return false; } } else { - $groupobject = $this->create_standard_group($course); + $groupobject = $this->create_standard_group($course, $baselevel + 1); if (!$groupobject) { return false; } } + if (isset($SESSION->o365_newly_created_groups)) { + $SESSION->o365_newly_created_groups[] = $groupobject['objectid']; + } + // Add owners / members to the group. $ownerobjectids = utils::get_team_owner_object_ids_by_course_id($course->id); $memberobjectids = utils::get_team_member_object_ids_by_course_id($course->id, $ownerobjectids); - $owneradded = $this->add_group_owners_and_members_to_group($groupobject['objectid'], $ownerobjectids, $memberobjectids); + $owneradded = $this->add_group_owners_and_members_to_group($groupobject['objectid'], $ownerobjectids, $memberobjectids, + $baselevel + 1); // If owner exists, create team. if ($owneradded) { // Owner exists, proceed with Team creation. if ($this->haseducationlicense) { - $this->create_class_team_from_education_group($groupobject['objectid'], $course); + $this->create_class_team_from_education_group($groupobject['objectid'], $course, $baselevel + 1); } else { - $this->create_team_from_standard_group($groupobject['objectid'], $course); + $this->create_team_from_standard_group($groupobject['objectid'], $course, $baselevel + 1); } } + $this->mtrace('Finished processing course #' . $course->id, $baselevel); + return true; } @@ -692,9 +888,9 @@ public function create_group_for_course(stdClass $course) : bool { * - Create Teams if appropriate. */ private function process_courses_without_teams() { - global $DB; + global $DB, $SESSION; - $this->mtrace('Processing courses without teams...'); + $this->mtrace('Process courses without teams...', 1); $sql = "SELECT crs.*, obj_group.objectid AS groupobjectid FROM {course} crs @@ -727,17 +923,49 @@ private function process_courses_without_teams() { foreach ($courses as $course) { if ($coursesprocessed > $courselimit) { - $this->mtrace('Course processing limit of ' . $courselimit . ' reached. Exit.'); + $this->mtrace('Course processing limit of ' . $courselimit . ' reached. Exit.', 1); + $this->mtrace('', 1); break; } - $this->mtrace('Processing course #' . $course->id, 1); + $this->mtrace('Process course #' . $course->id, 2); // Check if the team has owners. $owners = utils::get_team_owner_object_ids_by_course_id($course->id); $members = utils::get_team_member_object_ids_by_course_id($course->id, $owners); - if ($owners) { + // Verify that at least one owner exists. + if (isset($SESSION->o365_users_not_exist)) { + $owners = array_diff($owners, $SESSION->o365_users_not_exist); + $members = array_diff($members, $SESSION->o365_users_not_exist); + } + $ownerexists = false; + foreach ($owners as $owner) { + try { + $o365user = $this->graphclient->get_user($owner); + if ($o365user) { + $ownerexists = true; + break; + } else { + if (isset($SESSION->o365_users_not_exist)) { + if (!in_array($owner, $SESSION->o365_users_not_exist)) { + $SESSION->o365_users_not_exist[] = $owner; + } + } + } + } catch (moodle_exception $e) { + if (isset($SESSION->o365_users_not_exist)) { + if (static::is_resource_not_exist_exception($e->getMessage())) { + $useroid = \local_o365\utils::extract_guid_from_error_message($e->getMessage()); + if (!empty($useroid) && !in_array($useroid, $SESSION->o365_users_not_exist)) { + $SESSION->o365_users_not_exist[] = $useroid; + } + } + } + } + } + + if ($owners && $ownerexists) { // Resync group owners and members, just in case. $this->add_group_owners_and_members_to_group($course->groupobjectid, $owners, $members); if ($this->haseducationlicense) { @@ -751,13 +979,12 @@ private function process_courses_without_teams() { } } else { $this->mtrace('Skip creating team from group with ID ' . $course->groupobjectid . ' for course #' . $course->id . - '. Reason: No owner.', 2); + '. Reason: No owner.', 3); } } - if (empty($coursesprocessed)) { - $this->mtrace('All courses have teams created.', 1); - } else { + $this->mtrace('Finished processing courses without teams.', 1); + if ($coursesprocessed) { $this->mtrace('Created teams for ' . $coursesprocessed . ' courses.', 1); } @@ -804,7 +1031,7 @@ private function restore_group(int $objectrecid, string $objectid, array $object public function update_teams_cache() : bool { global $DB; - $this->mtrace('Updating teams cache...'); + $this->mtrace('Update teams cache...'); $coursesyncsetting = get_config('local_o365', 'coursesync'); if (!($coursesyncsetting === 'onall' || $coursesyncsetting === 'oncustom')) { @@ -877,6 +1104,9 @@ public function update_teams_cache() : bool { $DB->delete_records('local_o365_teams_cache', ['id' => $oldcacherecord->id]); } + $this->mtrace('Finished updating teams cache.'); + $this->mtrace(''); + // Set last updated timestamp. set_config('teamscacheupdated', time(), 'local_o365'); @@ -895,7 +1125,7 @@ public function cleanup_teams_connections() { $teamobjectids = $DB->get_fieldset_select('local_o365_teams_cache', 'objectid', ''); - $this->mtrace('Cleaning up teams connection records...'); + $this->mtrace('Clean up teams connection records...'); if ($teamobjectids) { // If there are records in teams cache, delete teams connection records with object IDs not in the cache. [$teamobjectidsql, $params] = $DB->get_in_or_equal($teamobjectids, SQL_PARAMS_QM, 'param', false); @@ -905,6 +1135,8 @@ public function cleanup_teams_connections() { // If there are no records in teams cache, delete all teams connection records. $DB->delete_records_select('local_o365_objects', "type = 'group' AND subtype IN ('classteam', 'teamfromgroup')"); } + $this->mtrace('Finished cleaning up teams connection records.'); + $this->mtrace(''); } /** @@ -916,7 +1148,7 @@ public function cleanup_teams_connections() { public function cleanup_course_connection_records() { global $DB; - $this->mtrace('Cleaning up duplicate course connection records...'); + $this->mtrace('Clean up duplicate course connection records...'); $sql = " SELECT * @@ -937,6 +1169,9 @@ public function cleanup_course_connection_records() { $DB->delete_records('local_o365_objects', ['id' => $courseconnectionrecord->id]); } } + + $this->mtrace('Finished cleaning up duplicate course connection records.'); + $this->mtrace(''); } /** @@ -1141,7 +1376,7 @@ public function process_course_team_user_sync_from_moodle_to_microsoft(int $cour } if ($skip) { - $this->mtrace('Skipping syncing group owners / members for course'); + $this->mtrace('Skipped syncing group owners / members for course ' . $courseid, 2); return false; } @@ -1506,7 +1741,7 @@ public function process_course_team_user_sync_from_microsoft_to_moodle(int $cour } if ($skip) { - $this->mtrace('Skipping syncing group owners / members for course'); + $this->mtrace('Skipped syncing group owners / members for course', 2); return false; } @@ -1551,7 +1786,7 @@ public function process_course_team_user_sync_from_microsoft_to_moodle(int $cour // - $connectedintendedcourseteachers contains the teachers that should be in the course. $teacherstoenrol = array_diff($connectedintendedcourseteachers, $connectedcurrentcourseteachers); if ($teacherstoenrol) { - $this->mtrace('Adding teacher role to ' . count($teacherstoenrol) . ' users...', 2); + $this->mtrace('Add teacher role to ' . count($teacherstoenrol) . ' users...', 2); foreach ($teacherstoenrol as $userid) { $this->assign_role_by_user_id_role_id_and_course_context($userid, $ownerroleid, $coursecontext); } @@ -1575,7 +1810,7 @@ public function process_course_team_user_sync_from_microsoft_to_moodle(int $cour // - $connectedintendedcoursestudents contains the students that should be in the course. $studentstoenrol = array_diff($connectedintendedcoursestudents, $connectedcurrentcoursestudents); if ($studentstoenrol) { - $this->mtrace('Adding student role to ' . count($studentstoenrol) . ' users...', 2); + $this->mtrace('Add student role to ' . count($studentstoenrol) . ' users...', 2); foreach ($studentstoenrol as $userid) { $this->assign_role_by_user_id_role_id_and_course_context($userid, $memberroleid, $coursecontext); } @@ -1746,7 +1981,7 @@ public function process_initial_course_team_user_sync(int $courseid, string $gro // - $connectedintendedcourseteachers contains the teachers that should be in the course. $teacherstoenrol = array_diff($connectedintendedcourseteachers, $connectedcurrentcourseteachers); if ($teacherstoenrol) { - $this->mtrace('Adding teacher role to ' . count($teacherstoenrol) . ' users...', 2); + $this->mtrace('Add teacher role to ' . count($teacherstoenrol) . ' users...', 2); foreach ($teacherstoenrol as $userid) { $this->assign_role_by_user_id_role_id_and_course_context($userid, $ownerroleid, $coursecontext); } @@ -1759,7 +1994,7 @@ public function process_initial_course_team_user_sync(int $courseid, string $gro // - $connectedintendedcoursestudents contains the students that should be in the course. $studentstoenrol = array_diff($connectedintendedcoursestudents, $connectedcurrentcoursestudents); if ($studentstoenrol) { - $this->mtrace('Adding student role to ' . count($studentstoenrol) . ' users...', 2); + $this->mtrace('Add student role to ' . count($studentstoenrol) . ' users...', 2); foreach ($studentstoenrol as $userid) { $this->assign_role_by_user_id_role_id_and_course_context($userid, $memberroleid, $coursecontext); } @@ -1781,4 +2016,35 @@ public function process_initial_course_team_user_sync(int $courseid, string $gro $this->mtrace('Add ' . count($owneroids) . ' owners and ' . count($memberoids) . ' members in bulk', 2); $this->add_group_owners_and_members_to_group($groupobjectid, $owneroids, $memberoids); } + + /** + * Save the non-existing groups to the database. + * + * @return void + */ + public function save_not_found_groups() : void { + global $DB, $SESSION; + + $this->mtrace('Save non-existing groups to groups cache...'); + if ($SESSION->o365_groups_not_exist) { + foreach ($SESSION->o365_groups_not_exist as $groupid) { + if ($existingrecord = $DB->get_record('local_o365_groups_cache', ['objectid' => $groupid])) { + if (!$existingrecord->not_found_since) { + $existingrecord->not_found_since = time(); + $DB->update_record('local_o365_groups_cache', $existingrecord); + $this->mtrace('Updated not found since value for group ' . $groupid . '.', 1); + } + } else { + $record = new stdClass(); + $record->objectid = $groupid; + $record->not_found_since = time(); + $DB->insert_record('local_o365_groups_cache', $record); + $this->mtrace('Created non-existing group ' . $groupid . ' and saved not found since value.', 1); + } + } + } + + $this->mtrace('Finished saving non-existing groups to groups cache.'); + $this->mtrace(''); + } } diff --git a/local/o365/classes/task/cohortsync.php b/local/o365/classes/task/cohortsync.php index 09b84cfae..847f81d17 100644 --- a/local/o365/classes/task/cohortsync.php +++ b/local/o365/classes/task/cohortsync.php @@ -29,6 +29,7 @@ use core\task\scheduled_task; use local_o365\feature\cohortsync\main; +use local_o365\utils; /** * A scheduled task to process Microsoft group and Moodle cohort mapping. @@ -51,7 +52,7 @@ public function get_name() : string { public function execute() : bool { $graphclient = main::get_unified_api(__METHOD__); if (empty($graphclient)) { - mtrace("... Failed to get Graph API client. Exiting."); + utils::mtrace("Failed to get Graph API client. Exiting.", 1); return true; } @@ -69,28 +70,31 @@ public function execute() : bool { * @return void */ private function execute_sync(main $cohortsync) : void { - if (!$cohortsync->update_groups_cache()) { - mtrace("... Failed to update groups cache. Exiting."); + if ($cohortsync->update_groups_cache()) { + utils::clean_up_not_found_groups(); + } else { + utils::mtrace("Failed to update groups cache. Exiting.", 1); return; } - mtrace("... Start processing cohort mappings."); + utils::mtrace("Start processing cohort mappings.", 1); $grouplist = $cohortsync->get_grouplist(); - mtrace("...... Found " . count($grouplist) . " groups."); + utils::mtrace("Found " . count($grouplist) . " groups.", 2); $grouplistbyoid = []; foreach ($grouplist as $group) { - $grouplistbyoid[$group['id']] = $group; + $grouplistbyoid[$group->objectid] = $group; } $mappings = $cohortsync->get_mappings(); if (empty($mappings)) { - mtrace("...... No mappings found. Nothing to process. Exiting."); + utils::mtrace("No mappings found. Nothing to process. Exiting.", 1); + utils::mtrace("", 1); return; } - mtrace("...... Found " . count($mappings) . " mappings."); + utils::mtrace("Found " . count($mappings) . " mappings.", 2); $cohorts = $cohortsync->get_cohortlist(); @@ -98,20 +102,20 @@ private function execute_sync(main $cohortsync) : void { // Verify that the group still exists. if (!in_array($mapping->objectid, array_keys($grouplistbyoid))) { $cohortsync->delete_mapping_by_group_oid_and_cohort_id($mapping->objectid, $mapping->moodleid); - mtrace("......... Deleted mapping for non-existing group ID {$mapping->objectid}."); + utils::mtrace("Deleted mapping for non-existing group ID {$mapping->objectid}.", 3); unset($mappings[$key]); } // Verify that the cohort still exists. if (!in_array($mapping->moodleid, array_keys($cohorts))) { $cohortsync->delete_mapping_by_group_oid_and_cohort_id($mapping->objectid, $mapping->moodleid); - mtrace("......... Deleted mapping for non-existing cohort ID {$mapping->moodleid}."); + utils::mtrace("Deleted mapping for non-existing cohort ID {$mapping->moodleid}.", 3); unset($mappings[$key]); } } foreach ($mappings as $mapping) { - mtrace("......... Processing mapping for group ID {$mapping->objectid} and cohort ID {$mapping->moodleid}."); + utils::mtrace("Processing mapping for group ID {$mapping->objectid} and cohort ID {$mapping->moodleid}.", 3); $cohortsync->sync_members_by_group_oid_and_cohort_id($mapping->objectid, $mapping->moodleid); } } diff --git a/local/o365/classes/task/coursesync.php b/local/o365/classes/task/coursesync.php index a5555c9c8..59b50271a 100644 --- a/local/o365/classes/task/coursesync.php +++ b/local/o365/classes/task/coursesync.php @@ -50,6 +50,12 @@ public function get_name() { * @return bool|void */ public function execute() { + global $SESSION; + + $SESSION->o365_groups_not_exist = []; + $SESSION->o365_newly_created_groups = []; + $SESSION->o365_users_not_exist = []; + if (utils::is_connected() !== true) { return false; } @@ -72,5 +78,10 @@ public function execute() { $coursesync->cleanup_teams_connections(); } $coursesync->cleanup_course_connection_records(); + + if (utils::update_groups_cache($graphclient, 1)) { + $coursesync->save_not_found_groups(); + utils::clean_up_not_found_groups(); + } } } diff --git a/local/o365/classes/utils.php b/local/o365/classes/utils.php index 06053f835..8165ccde3 100644 --- a/local/o365/classes/utils.php +++ b/local/o365/classes/utils.php @@ -35,6 +35,7 @@ use local_o365\obj\o365user; use local_o365\rest\unified; use moodle_exception; +use stdClass; defined('MOODLE_INTERNAL') || die(); @@ -44,6 +45,8 @@ * General purpose utility class. */ class utils { + const RESOURCE_NOT_EXIST_ERROR = 'does not exist or one of its queried reference-property objects are not present'; + /** * Determine whether essential configuration has been completed. * @@ -526,4 +529,133 @@ public static function get_connected_users() : array { return $connectedusers; } + + /** + * Update Groups cache. + * + * @param unified $graphclient + * @param int $baselevel + * @return bool + */ + public static function update_groups_cache(unified $graphclient, int $baselevel = 0) : bool { + global $DB; + + static::mtrace("Update groups cache.", $baselevel); + + try { + $grouplist = $graphclient->get_groups(); + } catch (moodle_exception $e) { + static::mtrace("Failed to fetch groups. Error: " . $e->getMessage(), $baselevel + 1); + + return false; + } + + $existingcacherecords = $DB->get_records('local_o365_groups_cache'); + $existinggroupsbyoid = []; + $existingnotfoundgroupsbyoid = []; + foreach ($existingcacherecords as $existingcacherecord) { + if ($existingcacherecord->not_found_since) { + $existingnotfoundgroupsbyoid[$existingcacherecord->objectid] = $existingcacherecord; + } else { + $existinggroupsbyoid[$existingcacherecord->objectid] = $existingcacherecord; + } + } + + foreach ($grouplist as $group) { + if (array_key_exists($group['id'], $existingnotfoundgroupsbyoid)) { + $cacherecord = $existingnotfoundgroupsbyoid[$group['id']]; + $cacherecord->name = $group['displayName']; + $cacherecord->description = $group['description']; + $cacherecord->not_found_since = 0; + $DB->update_record('local_o365_groups_cache', $cacherecord); + unset($existingnotfoundgroupsbyoid[$group['id']]); + static::mtrace("Unset not found flag for group {$group['id']}.", $baselevel + 1); + } else if (array_key_exists($group['id'], $existinggroupsbyoid)) { + $cacherecord = $existinggroupsbyoid[$group['id']]; + if ($cacherecord->name != $group['displayName'] || $cacherecord->description != $group['description']) { + $cacherecord->name = $group['displayName']; + $cacherecord->description = $group['description']; + $DB->update_record('local_o365_groups_cache', $cacherecord); + static::mtrace("Updated group ID {$group['id']} in cache.", $baselevel + 1); + } else { + static::mtrace("Group ID {$group['id']} in cache is up to date.", $baselevel + 1); + } + unset($existinggroupsbyoid[$group['id']]); + } else { + $cacherecord = new stdClass(); + $cacherecord->objectid = $group['id']; + $cacherecord->name = $group['displayName']; + $cacherecord->description = $group['description']; + $DB->insert_record('local_o365_groups_cache', $cacherecord); + static::mtrace("Added group ID {$group['id']} to cache.", $baselevel + 1); + } + } + + foreach ($existinggroupsbyoid as $oldcacherecord) { + $oldcacherecord->not_found_since = time(); + $DB->update_record('local_o365_groups_cache', $oldcacherecord); + static::mtrace("Marked group {$oldcacherecord->objectid} as not found in the cache.", $baselevel + 1); + } + + static::mtrace("Finished updating groups cache.", $baselevel); + static::mtrace("", $baselevel); + + return true; + } + + /** + * Clean up non-existing groups from the database. + * + * @param int $baselevel + * @return void + */ + public static function clean_up_not_found_groups(int $baselevel = 1) : void { + global $DB; + + static::mtrace('Clean up non-existing groups from database', $baselevel); + + $cutofftime = strtotime('-5 minutes'); + $sql = "SELECT * + FROM {local_o365_groups_cache} + WHERE not_found_since != 0 + AND not_found_since < :cutofftime"; + $records = $DB->get_records_sql($sql, ['cutofftime' => $cutofftime]); + + foreach ($records as $record) { + $DB->delete_records('local_o365_groups_cache', ['objectid' => $record->objectid]); + $DB->delete_records('local_o365_objects', ['objectid' => $record->objectid]); + $DB->delete_records('local_o365_teams_cache', ['objectid' => $record->objectid]); + static::mtrace('Deleted non-existing group ' . $record->objectid . ' from groups cache.', $baselevel + 1); + } + + static::mtrace('Finished cleaning up non-existing groups from database.', $baselevel); + static::mtrace('', $baselevel); + } + + /** + * Print a message to the debugging console. + * + * @param string $message + * @param int $level + * @param string $eol + * @return void + */ + public static function mtrace(string $message, int $level = 0, string $eol = "\n") { + if ($level) { + $message = str_repeat('...', $level) . ' ' . $message; + } + mtrace($message, $eol); + } + + /** + * Extract GUID from error message. + * + * @param string $errormessage + * @return string|null + */ + public static function extract_guid_from_error_message(string $errormessage) : ?string { + $pattern = '/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/'; + preg_match($pattern, $errormessage, $matches); + return $matches[0] ?? null; + } } diff --git a/local/o365/db/install.xml b/local/o365/db/install.xml index fe4a38d07..b9dff0414 100644 --- a/local/o365/db/install.xml +++ b/local/o365/db/install.xml @@ -1,5 +1,5 @@ - @@ -171,6 +171,7 @@ + diff --git a/local/o365/db/upgrade.php b/local/o365/db/upgrade.php index 3b7831011..3133deda5 100644 --- a/local/o365/db/upgrade.php +++ b/local/o365/db/upgrade.php @@ -990,8 +990,7 @@ function xmldb_local_o365_upgrade($oldversion) { try { $graphclient = main::get_unified_api(__METHOD__); if ($graphclient) { - $cohortsync = new main($graphclient); - $cohortsync->update_groups_cache(); + utils::update_groups_cache($graphclient); } } catch (moodle_exception $e) { // Do nothing. @@ -1188,5 +1187,19 @@ function xmldb_local_o365_upgrade($oldversion) { upgrade_plugin_savepoint(true, 2023100917, 'local', 'o365'); } + if ($oldversion < 2023100924) { + // Define field not_found_since to be added to local_o365_groups_cache. + $table = new xmldb_table('local_o365_groups_cache'); + $field = new xmldb_field('not_found_since', XMLDB_TYPE_INTEGER, '10', null, XMLDB_NOTNULL, null, '0', 'description'); + + // Conditionally launch add field not_found_since. + if (!$dbman->field_exists($table, $field)) { + $dbman->add_field($table, $field); + } + + // O365 savepoint reached. + upgrade_plugin_savepoint(true, 2023100924, 'local', 'o365'); + } + return true; } diff --git a/local/o365/version.php b/local/o365/version.php index b7b313522..39f2f471b 100644 --- a/local/o365/version.php +++ b/local/o365/version.php @@ -26,7 +26,7 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023100921; +$plugin->version = 2023100924; $plugin->requires = 2023100900; $plugin->release = '4.3.5'; $plugin->component = 'local_o365';