Add a command to generate missing mappings

.github/workflows/generate_missing_mappings.yml

@@ -0,0 +1,79 @@
+name: Generate and commit mapping changes to requested PR
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  generate_missing_mappings:
+    runs-on: shopify-ubuntu-latest
+    if: ${{github.event.issue.pull_request && github.event.comment.body == '/generate_mappings'}}
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Checkout PR
+      run: gh pr checkout $ISSUE
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        ISSUE: ${{ github.event.issue.html_url }}
+    - uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true
+    - name: Set up Podman
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install podman
+    - name: Launch Qdrant server
+      run: |
+        port=6333
+        podman run -d -p ${port}:${port} qdrant/qdrant
+        echo "Waiting for Qdrant server to be ready..."
+        sleep 25
+    - uses: cue-lang/[email protected]
+      with:
+        version: 'v0.7.0'
+    - name: Generate distribution files to gather all published mappings in in json format
+      run: make VERBOSE=1
+    - name: Generate missing mappings
+      env:
+        OPENAI_API_TOKEN: ${{ secrets.OPENAI_API_TOKEN }}
+      run: |
+        bin/generate_missing_mappings
+    - name: Clean cache and re-generate mapping distribution files
+      run: |
+        make clean
+        make VERBOSE=1
+    - name: Commit mapping changes
+      id: commit_changes
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git add dist/*/integrations
+        git add data/integrations
+        commit_output=$(git commit -m "🤖 Update missing mappings" || echo "No changes to commit")
+        echo "$commit_output"
+        echo "::set-output name=commit_message::$commit_output"
+        git push || echo "No changes to push"
+    - name: Read disagree mapping entries
+      id: read_content
+      run: |
+        if [ -f tmp/mapping_update_message.txt ]; then
+          meaningful_content=$(cat tmp/mapping_update_message.txt)
+        else
+          meaningful_content="No meaningful content provided."
+        fi
+        echo "meaningful_content<<EOF" >> $GITHUB_ENV
+        echo "$meaningful_content" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
+    - name: Add mapping grading comment to PR
+      if: steps.commit_changes.outputs.commit_message != 'No changes to commit'
+      uses: actions/github-script@v6
+      with:
+        script: |
+          const body = process.env.meaningful_content;
+          github.rest.issues.createComment({
+            issue_number: ${{ github.event.issue.number }},
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            body: body
+          })

Gemfile

@@ -19,6 +19,11 @@ gem "jekyll-redirect-from", "~> 0.16"
 gem "cli-ui", "~> 2.2"
 gem "tty-option", "~> 0.3"
 
+# generate taxonomy mappings
+gem "qdrant-ruby"
+gem "bigdecimal"
+gem "ruby-openai"
+
 group :development, :test do
   gem "debug", platforms: [:mri, :windows]
   gem "mocha"

Gemfile.lock

@@ -97,12 +97,20 @@ GEM
       eventmachine (>= 0.12.9)
       http_parser.rb (~> 0)
     erubi (1.12.0)
+    event_stream_parser (1.0.0)
     eventmachine (1.2.7)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
     factory_bot_rails (6.4.3)
       factory_bot (~> 6.4)
       railties (>= 5.0.0)
+    faraday (2.10.0)
+      faraday-net_http (>= 2.0, < 3.2)
+      logger
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.1.1)
+      net-http
     ffi (1.16.3)
     forwardable-extended (2.6.0)
     globalid (1.2.1)
@@ -158,6 +166,7 @@ GEM
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.6.0)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -175,7 +184,10 @@ GEM
     mocha (2.3.0)
       ruby2_keywords (>= 0.0.5)
     msgpack (1.7.2)
+    multipart-post (2.4.1)
     mutex_m (0.2.0)
+    net-http (0.4.1)
+      uri
     net-imap (0.4.11)
       date
       net-protocol
@@ -209,6 +221,8 @@ GEM
     public_suffix (5.0.5)
     puma (6.4.2)
       nio4r (~> 2.0)
+    qdrant-ruby (0.9.7)
+      faraday (>= 2.0.1, < 3)
     racc (1.8.0)
     rack (3.0.11)
     rack-session (2.0.0)
@@ -275,6 +289,10 @@ GEM
       parser (>= 3.3.1.0)
     rubocop-shopify (2.15.1)
       rubocop (~> 1.51)
+    ruby-openai (7.1.0)
+      event_stream_parser (>= 0.3.0, < 2.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
     safe_yaml (1.0.5)
@@ -314,6 +332,7 @@ GEM
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.5.0)
+    uri (0.13.0)
     webrick (1.8.1)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
@@ -337,6 +356,7 @@ PLATFORMS
   x86_64-linux-musl
 
 DEPENDENCIES
+  bigdecimal
   bootsnap
   cli-ui (~> 2.2)
   debug
@@ -346,8 +366,10 @@ DEPENDENCIES
   minitest-hooks (~> 1.5)
   mocha
   puma (>= 5.0)
+  qdrant-ruby
   rails (~> 7.1.3, >= 7.1.3.2)
   rubocop-shopify
+  ruby-openai
   sqlite3 (~> 1.7)
   tty-option (~> 0.3)
   tzinfo-data