Merge branch 'master' into task-3/fix-ruf012-classvar-annotations

.eslintrc.json

@@ -4,7 +4,7 @@
     "eslint-plugin-no-jquery"
   ],
   "extends": [
-    "plugin:vue/essential",
+    "plugin:vue/vue3-essential",
     "plugin:no-jquery/deprecated"
   ],
   "env": {

.github/workflows/pm_stale_ticket_labeler.yml

@@ -0,0 +1,24 @@
+name: Label inactive issues and pull requests
+on:
+  schedule:
+    - cron: "12 2 * * *"
+
+jobs:
+  label-stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          start-date: 2025-01-01
+          exempt-issue-labels: 'no-automation,"Type: Epic"'
+          exempt-assignees: jimchamp,mekarpeles,cdrini,scottbarnes,raybb,seabelis,hornc
+          days-before-close: -1  # signifies that neither issues nor pull requests are to be closed
+          days-before-issue-stale: 14
+          days-before-pr-stale: -1
+          include-only-assigned: true
+          stale-issue-label: "Needs: Review Assignee"
+          stale-issue-message: 'This issue has been marked "Needs: Review Assignee" since there have been no updates within the past two weeks'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,4 +1,4 @@
-openlibrary/components/_dev.js
+openlibrary/components/dev/_dev.js
 *venv
 *.pyc
 *~

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
       - id: auto-walrus
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.4
+    rev: v0.8.6
     hooks:
       - id: ruff
         args: [ --fix ]
@@ -56,7 +56,7 @@ repos:
     - id: cython-lint
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.0
+    rev: v1.14.1
     hooks:
       - id: mypy  # See pyproject.toml for args
         additional_dependencies:

Makefile

@@ -29,13 +29,13 @@ js:
 		echo "\n// @license-end"  >> $$js; \
 	done
 
-components: $(COMPONENTS_DIR)/*.vue
+components:
 	mkdir -p $(BUILD)
 	rm -rf $(BUILD)/components
-	# Run these silly things one at a time, because they don't support parallelization :(
-	parallel --verbose -q --jobs 1 \
-		npx vue-cli-service build --no-clean --mode production --dest $(BUILD)/components/production --target wc --name "ol-{/.}" "{}" \
-	::: $^
+	for comp in $$(find openlibrary/components -maxdepth 1 -name '*.vue' -printf "%f\n"); do \
+		COMPONENT="$$comp" npx vite build -c openlibrary/components/vite.config.mjs ; \
+	done
+
 
 i18n:
 	$(PYTHON) ./scripts/i18n-messages compile

compose.production.yaml

@@ -126,8 +126,6 @@ services:
     user: root
     command: docker/ol-cron-start.sh
     restart: unless-stopped
-    env_file:
-      - ../olsystem/etc/cron-jobs.env
     volumes:
       - ../olsystem/etc/cron.d/openlibrary.ol_home0:/etc/cron.d/openlibrary.ol_home0:ro
       - ../olsystem:/olsystem

docker/Dockerfile.olbase

@@ -38,22 +38,10 @@ RUN apt-get -qq update && apt-get install -y \
 COPY scripts/install_nodejs.sh ./
 RUN ./install_nodejs.sh && rm ./install_nodejs.sh
 
-# Install Archive.org nginx w/ IP anonymization
+# Install nginx
 USER root
-RUN apt-get update && apt-get install -y --no-install-recommends nginx curl letsencrypt \
-    # nginx-plus
-    apt-transport-https lsb-release ca-certificates wget \
-    # log rotation service for ol-nginx
-    logrotate \
-    # rsync service for pulling monthly sitemaps from ol-home0 to ol-www0
-    rsync
-COPY scripts/install_openresty.sh ./
-RUN ./install_openresty.sh && rm ./install_openresty.sh
-RUN rm /usr/sbin/nginx
-RUN curl -L https://archive.org/download/nginx/nginx -o /usr/sbin/nginx
-RUN chmod +x /usr/sbin/nginx
-# Remove the stock nginx config file
-RUN rm /etc/nginx/sites-enabled/default
+COPY scripts/install_nginx.sh ./
+RUN ./install_nginx.sh && rm ./install_nginx.sh
 
 RUN mkdir -p /var/log/openlibrary /var/lib/openlibrary && chown openlibrary:openlibrary /var/log/openlibrary /var/lib/openlibrary \
  && mkdir /openlibrary && chown openlibrary:openlibrary /openlibrary \

docker/covers_nginx.conf

@@ -14,6 +14,9 @@ server {
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
     ssl_prefer_server_ciphers on;
+
+    # Needed for logging/IP anonymization
+    include /olsystem/etc/nginx/logging_periodics.conf;
 }
 
 # Docker's internal load balancing ends up with unbalanced connections eventually.

docker/nginx.conf

@@ -1,3 +1,6 @@
+# Needed for IP anonymization
+load_module modules/ngx_http_js_module.so;
+
 user  www-data;
 
 # XXX-Anand: Oct 2013
@@ -25,7 +28,8 @@ http {
     server_names_hash_bucket_size   64;
     types_hash_bucket_size 64;
 
-    log_format iacombined '$remote_addr_ipscrub $host $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time';
+    # Logging / IP Anonymization; also need logging_periodics.conf inside a server block
+    include /olsystem/etc/nginx/logging.conf;
     access_log    /var/log/nginx/access.log iacombined;
 
     client_max_body_size 50m;

docker/web_nginx.conf

@@ -32,6 +32,9 @@ server {
     ssl_protocols TLSv1.2 TLSv1.3;
     ssl_ciphers EECDH+CHACHA20:EECDH+AES128:RSA+AES128:EECDH+AES256:RSA+AES256:EECDH+3DES:RSA+3DES:!MD5;
     ssl_prefer_server_ciphers on;
+
+    # Needed for logging/IP anonymization
+    include /olsystem/etc/nginx/logging_periodics.conf;
 }
 
 server {
@@ -65,6 +68,12 @@ server {
         rewrite ^(.*)$ https://$http_host$1 last;
     }
 
+    # Blue Sky
+    location /.well-known/atproto-did {
+        default_type text/plain;
+        return 200 'did:plc:gtjpwquxcbc5sygj5vcfefb3';
+    }
+
     # Return 429 errors as JSON.
     error_page 429 = @429;
     location @429 {

openlibrary/catalog/add_book/__init__.py

@@ -36,14 +36,15 @@
 from infogami import config
 from openlibrary import accounts
 from openlibrary.catalog.add_book.load_book import (
-    InvalidLanguage,
     build_query,
     east_in_by_statement,
     import_author,
 )
 from openlibrary.catalog.add_book.match import editions_match, mk_norm
 from openlibrary.catalog.utils import (
     EARLIEST_PUBLISH_YEAR_FOR_BOOKSELLERS,
+    InvalidLanguage,
+    format_languages,
     get_non_isbn_asin,
     get_publication_year,
     is_independently_published,
@@ -818,20 +819,30 @@ def update_edition_with_rec_data(
         'lc_classifications',
         'oclc_numbers',
         'source_records',
+        'languages',
     ]
-    for f in edition_list_fields:
-        if f not in rec or not rec[f]:
+    edition_dict: dict = edition.dict()
+    for field in edition_list_fields:
+        if field not in rec:
             continue
-        # ensure values is a list
-        values = rec[f] if isinstance(rec[f], list) else [rec[f]]
-        if f in edition:
-            # get values from rec field that are not currently on the edition
-            case_folded_values = {v.casefold() for v in edition[f]}
-            to_add = [v for v in values if v.casefold() not in case_folded_values]
-            edition[f] += to_add
+
+        existing_values = edition_dict.get(field, []) or []
+        rec_values = rec.get(field, [])
+
+        # Languages in `rec` are ['eng'], etc., but import requires dict-style.
+        if field == 'languages':
+            formatted_languages = format_languages(languages=rec_values)
+            supplemented_values = existing_values + [
+                lang for lang in formatted_languages if lang not in existing_values
+            ]
         else:
-            edition[f] = to_add = values
-        if to_add:
+            case_folded_values = [v.casefold() for v in existing_values]
+            supplemented_values = existing_values + [
+                v for v in rec_values if v.casefold() not in case_folded_values
+            ]
+
+        if existing_values != supplemented_values:
+            edition[field] = supplemented_values
             need_edition_save = True
 
     # Fields that are added as a whole if absent. (Individual values are not added.)
@@ -848,7 +859,7 @@ def update_edition_with_rec_data(
             edition[f] = rec[f]
             need_edition_save = True
 
-    # Add new identifiers
+    # Add new identifiers (dict values, so different treatment from lists above.)
     if 'identifiers' in rec:
         identifiers = defaultdict(list, edition.dict().get('identifiers', {}))
         for k, vals in rec['identifiers'].items():

openlibrary/catalog/add_book/load_book.py

@@ -2,7 +2,12 @@
 
 import web
 
-from openlibrary.catalog.utils import author_dates_match, flip_name, key_int
+from openlibrary.catalog.utils import (
+    author_dates_match,
+    flip_name,
+    format_languages,
+    key_int,
+)
 from openlibrary.core.helpers import extract_year
 
 if TYPE_CHECKING:
@@ -254,14 +259,6 @@ def import_author(author: dict[str, Any], eastern=False) -> "Author | dict[str,
     return a
 
 
-class InvalidLanguage(Exception):
-    def __init__(self, code):
-        self.code = code
-
-    def __str__(self):
-        return f"invalid language code: '{self.code}'"
-
-
 type_map = {'description': 'text', 'notes': 'text', 'number_of_pages': 'int'}
 
 
@@ -283,12 +280,12 @@ def build_query(rec: dict[str, Any]) -> dict[str, Any]:
                     east = east_in_by_statement(rec, author)
                     book['authors'].append(import_author(author, eastern=east))
             continue
+
         if k in ('languages', 'translated_from'):
-            for language in v:
-                if web.ctx.site.get('/languages/' + language.lower()) is None:
-                    raise InvalidLanguage(language.lower())
-            book[k] = [{'key': '/languages/' + language.lower()} for language in v]
+            formatted_languages = format_languages(languages=v)
+            book[k] = formatted_languages
             continue
+
         if k in type_map:
             t = '/type/' + type_map[k]
             if isinstance(v, list):