Merge pull request #23 from KentarouTakeda/content-negotiation

Fixed OGP acquisition failure due to Content Negotiation
h-sugawara · Jan 26, 2024 · de13bc8 · de13bc8
2 parents c0f7f4c + f4a231b
commit de13bc8
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/lib/parameters.js b/lib/parameters.js
@@ -33,7 +33,9 @@ function findOptionalArgs(opts, find) {
 
 function getFetchOptions(isCrawler) {
     const fetchOptions = {};
-    const headers = {};
+    const headers = {
+        'accept': 'text/html',
+    };
 
     if (typeof isCrawler === 'boolean' && isCrawler) {
         headers['user-agent'] = CRAWLER_USER_AGENT;

diff --git a/test/parameters.test.js b/test/parameters.test.js
@@ -15,6 +15,7 @@ describe('parameters', () => {
                     url: args[0],
                     fetchOptions: {
                         headers: {
+                            'accept': 'text/html',
                             'user-agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/112.0.0.0 Safari/537.36',
                         },
                     },
@@ -43,6 +44,7 @@ describe('parameters', () => {
                     url: args[0],
                     fetchOptions: {
                         headers: {
+                            'accept': 'text/html',
                             'user-agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/112.0.0.0 Safari/537.36',
                         },
                     },
@@ -60,7 +62,7 @@ describe('parameters', () => {
 
         expect(getParameters(args, fallbackText, config)).toEqual(
             {
-                scrape: { url: args[0], fetchOptions: {} },
+                scrape: { url: args[0], fetchOptions: { headers: { accept: 'text/html' } } },
                 generate: { target: '_blank', rel: 'nofollow', loading: 'lazy', descriptionLength, className, fallbackText },
             }
         );