Paginate users before making the request to avoid 414 (#507)

Making a single request with 1000 user ids, for example, is a large URL repeated many times in validation, which lead to 414 errors. Paginating the users and breaking this request down into many makes more sense because the number of requests won't change, and the URL won't be as big.
powerhome · Feb 7, 2025 · 8699837 · 8699837
1 parent 57f0f4f
commit 8699837
Show file tree

Hide file tree

Showing 9 changed files with 20 additions and 11 deletions.
diff --git a/audiences/Gemfile.lock b/audiences/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    audiences (1.5.3)
+    audiences (1.5.4)
       rails (>= 6.0)
 
 GEM
@@ -121,7 +121,6 @@ GEM
     marcel (1.0.4)
     method_source (1.1.0)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.8)
     minitest (5.25.4)
     mysql2 (0.5.6)
     net-imap (0.4.17)
@@ -134,8 +133,11 @@ GEM
     net-smtp (0.5.0)
       net-protocol
     nio4r (2.7.4)
-    nokogiri (1.16.8)
-      mini_portile2 (~> 2.8.2)
+    nokogiri (1.16.8-aarch64-linux)
+      racc (~> 1.4)
+    nokogiri (1.16.8-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.8-x86_64-linux)
       racc (~> 1.4)
     parallel (1.26.3)
     parser (3.3.6.0)

diff --git a/audiences/app/models/audiences/external_user.rb b/audiences/app/models/audiences/external_user.rb
@@ -10,11 +10,13 @@ class ExternalUser < ApplicationRecord
                             inverse_of: false
     end
 
-    def self.fetch(external_ids)
+    def self.fetch(external_ids, count: 100)
       return [] unless external_ids.any?
 
-      filter = Array(external_ids).map { "externalId eq #{_1}" }.join(" OR ")
-      Audiences::Scim.resource(:Users).all(filter: filter)
+      Array(external_ids).in_groups_of(count, false).flat_map do |ids|
+        filter = Array(ids).map { "externalId eq #{_1}" }.join(" OR ")
+        Audiences::Scim.resource(:Users).all(count: count, filter: filter).to_a
+      end
     end
 
     def self.wrap(resources)

diff --git a/audiences/docs/CHANGELOG.md b/audiences/docs/CHANGELOG.md
@@ -1,6 +1,9 @@
 # Unreleased
 
+# Version 1.5.4 (2024-12-19)
+
 - Fix `authenticate` / `authentication` configuration [#481](https://github.com/powerhome/audiences/pull/481)
+- Paginage user requests in audiences instead of SCIM [#507](https://github.com/powerhome/audiences/pull/507)
 
 # Version 1.5.3 (2024-12-19)
 

diff --git a/audiences/gemfiles/rails_6_1.gemfile.lock b/audiences/gemfiles/rails_6_1.gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    audiences (1.5.3)
+    audiences (1.5.4)
       rails (>= 6.0)
 
 GEM

diff --git a/audiences/gemfiles/rails_7_0.gemfile.lock b/audiences/gemfiles/rails_7_0.gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    audiences (1.5.3)
+    audiences (1.5.4)
       rails (>= 6.0)
 
 GEM

diff --git a/audiences/gemfiles/rails_7_1.gemfile.lock b/audiences/gemfiles/rails_7_1.gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    audiences (1.5.3)
+    audiences (1.5.4)
       rails (>= 6.0)
 
 GEM

diff --git a/audiences/lib/audiences/version.rb b/audiences/lib/audiences/version.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Audiences
-  VERSION = "1.5.3"
+  VERSION = "1.5.4"
 end
diff --git a/audiences/spec/controllers/contexts_controller_spec.rb b/audiences/spec/controllers/contexts_controller_spec.rb
@@ -51,6 +51,7 @@
       stub_request(:get, "http://example.com/scim/v2/Users")
         .with(query: {
                 attributes: "id,externalId,displayName,active,photos.type,photos.value",
+                count: 100,
                 filter: "(active eq true) and (externalId eq 123)",
               })
         .to_return(status: 200, body: { "Resources" => [{ "displayName" => "John Doe", "confidential" => "data",

diff --git a/audiences/spec/lib/audiences_spec.rb b/audiences/spec/lib/audiences_spec.rb
@@ -21,6 +21,7 @@
       stub_request(:get, "http://example.com/scim/v2/Users")
         .with(query: {
                 attributes: "id,externalId,displayName,active,photos.type,photos.value",
+                count: 100,
                 filter: "(active eq true) and (externalId eq 678 OR externalId eq 321)",
               })
         .to_return(status: 200, body: { "Resources" => [{ "displayName" => "John", "externalId" => 678 },