Fix parsing substitutions

* Fix outdated check for visible table lines * Fix handling documents with missing dates * Fix handling new Dropbox URL formats * Fix detecting non-substitutions data
filips123 · Aug 30, 2023 · 3f56dfc · 3f56dfc
1 parent d66df33
commit 3f56dfc
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 6 deletions.
diff --git a/API/gimvicurnik/updaters/eclassroom.py b/API/gimvicurnik/updaters/eclassroom.py
@@ -113,12 +113,24 @@ def _get_internal_urls(self) -> Iterator[DocumentInfo]:
 
                 url = self.normalize_url(module["contents"][0]["fileurl"])
 
+                modified = (
+                    datetime.fromtimestamp(module["contents"][0]["timemodified"], tz=timezone.utc)
+                    if module["contents"][0]["timemodified"]
+                    else None
+                )
+
+                created = (
+                    datetime.fromtimestamp(module["contents"][0]["timecreated"], tz=timezone.utc)
+                    if module["contents"][0]["timecreated"]
+                    else modified
+                )
+
                 yield DocumentInfo(
                     url=url,
                     type=self._get_document_type(url),
                     title=module["name"],
-                    created=datetime.fromtimestamp(module["contents"][0]["timecreated"], tz=timezone.utc),
-                    modified=datetime.fromtimestamp(module["contents"][0]["timemodified"], tz=timezone.utc),
+                    created=created,
+                    modified=modified,
                     extension=os.path.splitext(urlparse(url).path)[1][1:],
                 )
 
@@ -153,12 +165,18 @@ def _get_external_urls(self) -> Iterator[DocumentInfo]:
 
             url = self.normalize_url(content["externalurl"])
 
+            modified = (
+                datetime.fromtimestamp(content["timemodified"], tz=timezone.utc)
+                if content["timemodified"]
+                else None
+            )
+
             yield DocumentInfo(
                 url=url,
                 type=self._get_document_type(url),
                 title=content["name"],
-                created=datetime.fromtimestamp(content["timemodified"], tz=timezone.utc),
-                modified=datetime.fromtimestamp(content["timemodified"], tz=timezone.utc),
+                created=modified,
+                modified=modified,
                 extension=os.path.splitext(urlparse(url).path)[1][1:],
             )
 
@@ -182,6 +200,8 @@ def normalize_url(self, url: str) -> str:
             url.replace(self.config.pluginFileWebserviceUrl, self.config.pluginFileNormalUrl)
             .replace("?forcedownload=1", "")
             .replace("?dl=0", "?raw=1")
+            .replace("&dl=0", "&raw=1")
+            .replace("?rlkey=", "?raw=1&rlkey=")
         )
 
     def tokenize_url(self, url: str) -> str:
@@ -438,7 +458,7 @@ def _parse_substitutions(self, tables: Tables, effective: date) -> None:
                 elif row == header_reservations:
                     parser_type = ParserType.RESERVATIONS
                     continue
-                elif "Oddelek" in row[0] or "Razred" in row[0] or "dijaki" in row[0]:
+                elif "Oddelek" in row[0] or "Razred" in row[0] or "dijaki" in row[0] or "RAZREDNIK" in row[1]:
                     parser_type = ParserType.UNKNOWN
                     continue
 

diff --git a/API/gimvicurnik/utils/pdf.py b/API/gimvicurnik/utils/pdf.py
@@ -13,7 +13,7 @@
 
 def keep_visible_lines(obj: dict[str, Any]) -> bool:
     if obj["object_type"] == "rect":
-        visible: bool = obj["non_stroking_color"] == [0, 0, 0]
+        visible: bool = obj["non_stroking_color"] == (0, 0, 0)
         return visible
     return True