Merge remote-tracking branch 'refs/remotes/origin/main'

.github/workflows/build.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -47,6 +47,11 @@ jobs:
 #       pip install sphinx_rtd_theme
 #       scripts/doc
 
+    - name: Prepare cache directory
+      run: |
+        mkdir -p ~/.ceurws
+        chmod 755 ~/.ceurws
+
     - name: Cache Sqlite
       id: cache-sqlite
       uses: actions/cache@v4

ceurws/__init__.py

@@ -1 +1 @@
-__version__ = "0.4.1"
+__version__ = "0.4.2"

ceurws/resources/queries/ceurws.yaml

@@ -125,3 +125,20 @@
     where acronymlen < 20
     group by acronymlen
     order by 2 desc
+'WikidataSync':
+  sql: |
+    -- Volumes in Proceedings not in Volumes (no title, so we don't include these)
+    SELECT p.sVolume AS VolumeNumber, NULL AS Title, NULL AS Valid, 
+           strftime('%Y-%m-%d', datetime(p.publication_date / 1000000, 'unixepoch')) AS PublicationDate
+    FROM Proceedings p
+    LEFT JOIN volumes v ON p.sVolume = v.number
+    WHERE v.number IS NULL
+    
+    UNION
+    
+    -- Volumes in Volumes not in Proceedings (include title, valid flag, and publication date only when title is not NULL)
+    SELECT v.number AS VolumeNumber, v.title AS Title, v.valid AS Valid, 
+           strftime('%Y-%m-%d', datetime(v.pubDate / 1000000, 'unixepoch')) AS PublicationDate
+    FROM volumes v
+    LEFT JOIN Proceedings p ON v.number = p.sVolume
+    WHERE p.sVolume IS NULL AND v.title IS NOT NULL;

ceurws/version.py

@@ -18,7 +18,7 @@ class Version:
     name = "CEUR-WS Volume Browser"
     version = ceurws.__version__
     date = "2022-08-14"
-    updated = "2024-03-13"
+    updated = "2024-06-11"
     description = "CEUR-WS Volume browser"
 
     authors = "Tim Holzheim, Wolfgang Fahl"

ceurws/view.py

@@ -76,7 +76,7 @@ def createExternalLink(
             str - html link for external id
         """
         value = self.getRowValue(row, key)
-        if not value:
+        if not value or value==View.noneValue:
             if emptyIfNone:
                 return ""
             else:

ceurws/volume_view.py

@@ -9,7 +9,7 @@
 from ngwidgets.lod_grid import GridConfig, ListOfDictsGrid
 from ngwidgets.progress import NiceguiProgressbar
 from ngwidgets.widgets import Link
-from nicegui import ui
+from nicegui import ui, run
 
 from ceurws.ceur_ws import Volume
 from ceurws.view import View
@@ -241,9 +241,8 @@ def setup_ui(self):
                 self.ignore_errors_check_box = ui.checkbox("ignore_errors", value=self.ignore_errors).bind_value(
                     self, "ignore_errors"
                 )
-
                 pass
-            self.progress_bar = NiceguiProgressbar(total=100, desc="added", unit="volume")
+                self.progress_bar = NiceguiProgressbar(total=100, desc="added", unit="volume")
             with ui.row() as self.log_row:
                 self.log_view = ui.html()
             with ui.row() as self.grid_row:
@@ -264,7 +263,8 @@ def clear_msg(self, msg: str = ""):
         Args:
             msg(str): the message to display
         """
-        self.log_view.content = msg
+        with self.log_row:
+            self.log_view.content = msg
 
     def add_msg(self, html_markup: str):
         """
@@ -273,31 +273,41 @@ def add_msg(self, html_markup: str):
         Args:
             msg(str): the html formatted message to add
         """
-        self.log_view.content += html_markup
-
-    async def onWikidataButtonClick(self, _args):
+        with self.log_row:
+            self.log_view.content += html_markup
+
+    def updateWikidataVolumes(self,selected_rows):
         """
-        handle wikidata sync request
+        update wikidata volumes for the selected rows
         """
         try:
-            selected_rows = await self.lod_grid.get_selected_rows()
-            for row in selected_rows:
+            msg = f"{len(selected_rows)} Volumes selected<br>"
+            self.clear_msg(msg)
+            # First, sort selected_rows by the volume number in ascending order
+            sorted_rows = sorted(selected_rows, key=lambda row: row["#"])
+            for row in sorted_rows:
                 vol_number = row["#"]
-                volume = self.wdSync.volumesByNumber[vol_number]
-                msg = f"{len(selected_rows)} Volumes selected<br>"
-                self.clear_msg(msg)
-                await self.add_or_update_volume_in_wikidata(volume)
+                volume = self.wdSync.volumesByNumber[vol_number] 
+                self.add_or_update_volume_in_wikidata(volume)
             pass
         except Exception as ex:
             self.solution.handle_exception(ex)
 
-    async def on_check_recently_update_volumes_button_click(self, args):
+    async def onWikidataButtonClick(self, _args):
         """
-        handle clicking of the refresh button to get recently added volumes
+        handle wikidata sync request
+        """
+        selected_rows = await self.lod_grid.get_selected_rows()
+        await run.io_bound(self.updateWikidataVolumes,selected_rows)
+
+
+    def check_recently_updated_volumes(self):
+        """
+        check recently updated volumes
         """
         try:
             text = "checking CEUR-WS index.html for recently added volumes ..."
-            self.log_view.content = text
+            self.clear_msg(text)
             (
                 volumesByNumber,
                 addedVolumeNumberList,
@@ -317,11 +327,20 @@ async def on_check_recently_update_volumes_button_click(self, args):
                 self.add_msg(f":{link}")
             pass
             self.wdSync.storeVolumes()
-            self.progress_bar.reset()
-            self.lod_grid.update()
+            with self.parent:
+                self.progress_bar.reset()
+            with self.grid_row:
+                self.lod_grid.update()
         except Exception as ex:
             self.solution.handle_exception(ex)
 
+    async def on_check_recently_update_volumes_button_click(self, args):
+        """
+        handle clicking of the refresh button to get recently added volumes
+        """
+        await run.io_bound(self.check_recently_updated_volumes)
+
+
     def updateRecentlyAddedVolume(self, volume, index, total):
         """
         update a recently added Volume
@@ -359,18 +378,26 @@ def get_volume_lod(self):
                 }
             )
 
-    async def add_or_update_volume_in_wikidata(self, volume: Volume):
+    def add_or_update_volume_in_wikidata(self, volume: Volume):
+        """
+        add the given volume to wikidata or update it if it already exists
+        
+        Args:
+            volume(Volume): the CEUR-WS volume to update proceedings and event entries for
+        """
         try:
             msg = f"trying to add Volume {volume.number} to wikidata"
-            ui.notify(msg)
+            with self.parent:
+                ui.notify(msg)
             self.add_msg(msg + "<br>")
-            proceedingsWikidataId = await self.createProceedingsItemFromVolume(volume)
+            proceedingsWikidataId = self.createProceedingsItemFromVolume(volume)
             if proceedingsWikidataId is not None:
-                await self.createEventItemAndLinkProceedings(volume, proceedingsWikidataId)
+                self.createEventItemAndLinkProceedings(volume, proceedingsWikidataId)
             else:
                 msg = f"<br>adding Volume {volume.number} proceedings to wikidata failed"
                 self.add_msg(msg)
-                ui.notify(msg)
+                with self.parent:
+                    ui.notify(msg)
         except Exception as ex:
             self.solution.handle_exception(ex)
 
@@ -386,7 +413,7 @@ def optional_login(self) -> bool:
             self.wdSync.login()
         return write
 
-    async def createProceedingsItemFromVolume(self, volume: Volume):
+    def createProceedingsItemFromVolume(self, volume: Volume):
         """
         Create wikidata item for proceedings of given volume
         """
@@ -428,7 +455,7 @@ async def createProceedingsItemFromVolume(self, volume: Volume):
             self.solution.handle_exception(ex)
         return qId
 
-    async def createEventItemAndLinkProceedings(self, volume: Volume, proceedingsWikidataId: str | None = None):
+    def createEventItemAndLinkProceedings(self, volume: Volume, proceedingsWikidataId: str | None = None):
         """
         Create event  wikidata item for given volume and link
         the proceedings with the event

ceurws/wikidata_view.py

@@ -5,7 +5,7 @@
 """
 
 from ngwidgets.lod_grid import ListOfDictsGrid
-from nicegui import ui
+from nicegui import ui,run
 from wd.query_view import QueryView
 
 from ceurws.view import View
@@ -75,7 +75,7 @@ def reload_aggrid(self, olod: list):
                 row,
                 "ppnId",
                 "k10plus",
-                "https://opac.k10plus.de/DB=2.299/PPNSET?PPN=",
+                "https://opac.k10plus.de/DB=2.299/PPNSET?PPN="
             )
             lod.append(
                 {
@@ -92,16 +92,26 @@ def reload_aggrid(self, olod: list):
                 }
             )
         self.lod_grid.load_lod(lod)
+        # set max width of Item column
+        self.lod_grid.set_column_def("item","maxWidth",380)
+        self.lod_grid.set_column_def("event","maxWidth",380)
         self.lod_grid.sizeColumnsToFit()
-
+    
     async def on_refresh_button_click(self):
         """
         handle the refreshing of the proceedings from wikidata
         """
+        await run.io_bound(self.refresh_wikidata)
+
+    def refresh_wikidata(self):
         try:
-            ui.notify("wikidata refresh button clicked")
+            with self.solution.container:
+                ui.notify("wikidata refresh button clicked")
             wd_records = self.solution.wdSync.update()
-            self.lod_grid.load_lod(wd_records)
+            with self.solution.container:
+                ui.notify(f"read {len(wd_records)} proceeding records from wikidata")
+            with self.parent:
+                self.reload_aggrid(wd_records)
             pass
         except Exception as ex:
             self.solution.handle_exception(ex)

ceurws/wikidatasync.py

@@ -49,8 +49,9 @@ def __init__(
         self.wdQuery = self.qm.queriesByName["Proceedings"]
         self.baseurl = baseurl
         self.wd = Wikidata(debug=debug)
-        self.sqldb = SQLDB(CEURWS.CACHE_FILE)
+        self.sqldb = SQLDB(CEURWS.CACHE_FILE,check_same_thread=False)
         self.procRecords = None
+        self.procsByVolnumber = None
         self.dblpEndpoint = DblpEndpoint(endpoint=dblp_endpoint_url)
         self.wikidata_endpoint: Endpoint | None = None
 
@@ -294,6 +295,7 @@ def getProceedingsForVolume(self, searchVolnumber: int) -> dict | None:
         """
         if self.procRecords is None:
             self.loadProceedingsFromCache()
+        if self.procsByVolnumber is None:
             self.procsByVolnumber: dict[int, dict] = {}
             if isinstance(self.procRecords, list):
                 for procRecord in self.procRecords:

pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
     #https://github.com/martinblech/xmltodict
     'xmltodict>=0.13.0',
     # https://pypi.org/project/pylodstorage/
-    'pylodstorage>=0.11.5',
+    'pylodstorage>=0.11.6',
     # https://github.com/pyparsing/pyparsing
     'pyparsing>=3.0.9',
     # https://pypi.org/project/beautifulsoup4/
@@ -45,7 +45,7 @@ dependencies = [
     # https://pypi.org/project/neo4j/
     'neo4j',
     # https://github.com/WolfgangFahl/nicegui_widgets
-    'ngwidgets>=0.12.5',
+    'ngwidgets>=0.15.2',
     # https://pypi.org/project/sqlmodel/
     'sqlmodel>=0.0.16',
     # https://pypi.org/project/wdgrid/