Remove api-spec indexing

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Removed
+
+- Removed indexing of api-spec content (REST API specification)
+
 ## [3.4.2] - 2024-03-13
 
 ## [3.4.1] - 2024-02-13

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-alpine3.18
+FROM python:3.12-alpine3.20
 
 ENV PYTHON_UNBUFFERED 1
 ENV PYTHONWARNINGS "ignore:Unverified HTTPS request"

docs/configuration.md

@@ -14,9 +14,6 @@ The following environment variables are accepted, by indexer sub command:
 - `REPOSITORY_BRANCH`: Defaults to `main`.
 - `REPOSITORY_SUBFOLDER`: Only look into this path within the repository for indexable content.
 - `TYPE_LABEL`: User friendly search result type name.
-- `APIDOCS_BASE_URI`: Base URI for API documentation. Should be `https://docs.giantswarm.io/api/`.
-- `APIDOCS_BASE_PATH`: Should be `/api/`
-- `API_SPEC_FILES`: Comma separated list of YAML files to fetch for the OpenAPI spec.
 
 ## `blog`
 

hugo.py

@@ -3,7 +3,6 @@
 from opensearchpy.exceptions import NotFoundError
 from markdown import markdown
 from subprocess import call, check_output, STDOUT
-from prance import ResolvingParser
 import git
 import json
 import logging
@@ -12,7 +11,6 @@
 import shutil
 import sys
 import time
-import tempfile
 import urllib3
 import yaml
 
@@ -45,11 +43,6 @@
 # TODO: validate
 TYPE_LABEL = os.getenv("TYPE_LABEL")
 
-# TODO: remove
-APIDOCS_BASE_URI = os.getenv("APIDOCS_BASE_URI")
-APIDOCS_BASE_PATH = os.getenv("APIDOCS_BASE_PATH")
-API_SPEC_FILES = os.getenv("API_SPEC_FILES")
-
 WORKDIR = os.getenv("WORKDIR", "/home/indexer")
 
 # Path to markdown files
@@ -271,93 +264,12 @@ def index_page(osclient, root_path, path, breadcrumb, uri, index, last_modified)
         logging.error(f'Error when indexing page {uri}: {e}')
 
 
-def index_openapi_spec(osclient, uri, base_path, spec_files, index):
-    """
-    Indexes our API docs based on the Open API specification YAML
-
-    Params:
-
-    osclient: opensearch.OpenSearch client instance
-    """
-    files = spec_files.split(",")
-    tmpdir = tempfile.mkdtemp()
-
-    http = urllib3.PoolManager()
-
-    # download spec files
-    for filename in files:
-        url = uri + filename
-        logging.info("Reading URL %s" % url)
-        req = http.request("GET", url)
-        path = tmpdir + os.path.sep + os.path.basename(filename)
-        with open(path, "w") as outfile:
-            outfile.write(req.data.decode())
-
-    # parse spec
-    parser = ResolvingParser(tmpdir + os.path.sep + os.path.basename(files[0]))
-
-    for path in parser.specification["paths"]:
-        for method in parser.specification["paths"][path]:
-
-            target_uri = base_path + "#operation/" + parser.specification["paths"][path][method]["operationId"]
-
-            logging.info("Indexing %s %s as URL %s" % (method.upper(), path, target_uri))
-
-            title = u"%s - %s %s" % (parser.specification["paths"][path][method]["summary"],
-                method.upper(), path)
-
-            # forming body from operation spec
-            body = u"The %s API operation\n\n" % parser.specification["paths"][path][method]["operationId"]
-            body += markdown_to_text(parser.specification["paths"][path][method]["description"])
-            for code in parser.specification["paths"][path][method]["responses"]:
-                body += u"\n- %s" % parser.specification["paths"][path][method]["responses"][code]["description"]
-
-            text = title + "\n\n" + body
-
-            # breadcrumb (list of path segments) from base_path
-            parts = base_path.split("/")
-            breadcrumb = []
-            for part in parts:
-                if part != "":
-                    breadcrumb.append(part)
-            breadcrumb.append("#operation/" + parser.specification["paths"][path][method]["operationId"])
-
-            data = {
-                "title": title,
-                "uri": target_uri,
-                "breadcrumb": breadcrumb,
-                "body": body,
-                "text": text,
-                "date": DEFAULT_DATE.isoformat() + "+00:00",
-            }
-
-            osclient.index(
-                index=index,
-                id=data["uri"],
-                body=data)
-
-
 def read_crd(path):
     with open(path, "rb") as crdfile:
         crd = yaml.load(crdfile, Loader=Loader)
         return crd
 
 
-def collect_properties_text(schema_dict):
-    """
-    Recurses into an OpenAPIv3 hierarchy and returns property data valueable for full text indexing.
-    That's mainly the property name and a description, if present.
-    """
-    ret = []
-    if "description" in schema_dict:
-        ret.append(schema_dict["description"])
-    if "properties" in schema_dict:
-        for prop in schema_dict["properties"].keys():
-            ret.append(prop)
-            ret.extend(collect_properties_text(schema_dict["properties"][prop]))
-    return ret
-
-
 def check_index(es, index_name):
     """
     Check if the index already exists
@@ -380,7 +292,7 @@ def ensure_index(es, index_name):
 
 def run():
     """
-    Main function executing docs and api-spec indexing
+    Main function executing docs indexing
     """
     http = urllib3.PoolManager()
     url = f'https://api.github.com/repos/{REPOSITORY_HANDLE}/commits/{REPOSITORY_BRANCH}'
@@ -446,10 +358,6 @@ def run():
     # create new index
     ensure_index(osclient, full_index_name)
 
-    # index API spec
-    if (APIDOCS_BASE_URI is not None) and (APIDOCS_BASE_PATH is not None) and (API_SPEC_FILES is not None):
-        index_openapi_spec(osclient, APIDOCS_BASE_URI, APIDOCS_BASE_PATH, API_SPEC_FILES, full_index_name)
-
     # index docs pages
     for page in pages:
         index_page(osclient, main_path, page["file_path"], page["path"], page["uri"], full_index_name, last_modified)

requirements.txt

@@ -7,7 +7,6 @@ GitPython==3.1.43
 idna==3.7
 Markdown==3.6
 openapi-spec-validator==0.7.1
-prance==23.6.21.0
 requests==2.32.3
 semver==3.0.2
 six==1.16.0