From de71c7af92781334252e85d76d6b94f74b013930 Mon Sep 17 00:00:00 2001 From: Anthony Shull Date: Wed, 20 Dec 2023 13:57:10 -0600 Subject: [PATCH] Centralized CMS caching 1 (#1830) * just enough of the cms caching pr to prove that redis is working * fix mix lock * test that logger is working for endpoints * convert to test cache * set log level to notice * catch redis connection error and log * PR feedback * empty line * make test that redis disconnect is logged * default on basic auth for testing * force ssl * try to go through api * missing dep * reset router info * just slap those creds in there * runtime config issue for prod * log basic auth so we can make sure what we are getting from drupal * more deployment checking changes * hard code redis host to see if connection works * remove hard coded redis host --- .envrc.template | 8 ++++++ apps/alerts/config/runtime.exs | 5 ---- apps/cms/lib/cache.ex | 5 ++++ apps/cms/lib/cms.ex | 1 + apps/cms/mix.exs | 14 ++++++---- apps/site/config/config.exs | 4 +-- apps/site/config/dev.exs | 2 +- apps/site/config/prod.exs | 2 +- apps/site/config/test.exs | 5 ++-- .../site_web/controllers/cms_controller.ex | 28 +++++++++++++++++++ apps/site/lib/site_web/router.ex | 13 +++++++++ .../controllers/cms_controller_test.exs | 13 +++++++++ config/runtime.exs | 27 ++++++++++++++++++ mix.lock | 3 ++ rel/config.exs | 2 +- 15 files changed, 113 insertions(+), 19 deletions(-) delete mode 100644 apps/alerts/config/runtime.exs create mode 100644 apps/cms/lib/cache.ex create mode 100644 config/runtime.exs diff --git a/.envrc.template b/.envrc.template index bd3d2d3c62..99a1ddaaf9 100644 --- a/.envrc.template +++ b/.envrc.template @@ -29,3 +29,11 @@ export WIREMOCK_TRIP_PLAN_PROXY_URL=http://otp-local.mbtace.com # You can also log in manually with `docker login`. # export DOCKER_USERNAME= # export DOCKER_PASSWORD= + +# You can optionally set a Redis host. It will default to 127.0.0.1 +# export REDIS_HOST= + +# These credentials control access to resetting cache entries for the CMS. +# You can set them to be whatever you want, but they'll need to match those on the Drupal side. +# export CMS_BASIC_AUTH_USERNAME= +# export CMS_BASIC_AUTH_PASSWORD= diff --git a/apps/alerts/config/runtime.exs b/apps/alerts/config/runtime.exs deleted file mode 100644 index 35d4527e47..0000000000 --- a/apps/alerts/config/runtime.exs +++ /dev/null @@ -1,5 +0,0 @@ -import Config - -if config_env() == :prod do - config :alerts, bus_stop_change_bucket: System.get_env("S3_PREFIX_BUSCHANGE") -end diff --git a/apps/cms/lib/cache.ex b/apps/cms/lib/cache.ex new file mode 100644 index 0000000000..4303874968 --- /dev/null +++ b/apps/cms/lib/cache.ex @@ -0,0 +1,5 @@ +defmodule CMS.Cache do + use Nebulex.Cache, + otp_app: :cms, + adapter: NebulexRedisAdapter +end diff --git a/apps/cms/lib/cms.ex b/apps/cms/lib/cms.ex index 106dc23e3c..9ba675312c 100644 --- a/apps/cms/lib/cms.ex +++ b/apps/cms/lib/cms.ex @@ -11,6 +11,7 @@ defmodule CMS do # Define workers and child supervisors to be supervised children = [ + CMS.Cache, CMS.Repo ] diff --git a/apps/cms/mix.exs b/apps/cms/mix.exs index 97c61e54bb..54f6dd9441 100644 --- a/apps/cms/mix.exs +++ b/apps/cms/mix.exs @@ -47,16 +47,18 @@ defmodule CMS.Mixfile do # Type "mix help deps" for more examples and options defp deps do [ - {:httpoison, ">= 0.0.0"}, - {:poison, ">= 0.0.0", override: true}, - {:timex, ">= 0.0.0"}, - {:plug, "~> 1.14.2"}, - {:html_sanitize_ex, "1.3.0"}, {:bypass, "~> 1.0", only: :test}, - {:quixir, "~> 0.9", only: :test}, + {:html_sanitize_ex, "1.3.0"}, + {:httpoison, ">= 0.0.0"}, {:mock, "~> 0.3.3", only: :test}, + {:nebulex, "2.5.2"}, + {:nebulex_redis_adapter, "2.3.1"}, {:phoenix_html, "~> 3.0"}, + {:plug, "~> 1.14.2"}, + {:poison, ">= 0.0.0", override: true}, + {:quixir, "~> 0.9", only: :test}, {:repo_cache, in_umbrella: true}, + {:timex, ">= 0.0.0"}, {:util, in_umbrella: true} ] end diff --git a/apps/site/config/config.exs b/apps/site/config/config.exs index 37c93f7f9d..c7c3a5ea14 100644 --- a/apps/site/config/config.exs +++ b/apps/site/config/config.exs @@ -3,7 +3,7 @@ # # This configuration file is loaded before any dependency and # is restricted to this project. -use Mix.Config +import Config # Configures the endpoint config :site, SiteWeb.Endpoint, @@ -81,4 +81,4 @@ config :site, # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/apps/site/config/dev.exs b/apps/site/config/dev.exs index 44cdcac38b..ede858950b 100644 --- a/apps/site/config/dev.exs +++ b/apps/site/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For development, we disable any cache and enable # debugging and code reloading. diff --git a/apps/site/config/prod.exs b/apps/site/config/prod.exs index 07787b9fc4..10db90ef0e 100644 --- a/apps/site/config/prod.exs +++ b/apps/site/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For production, we configure the host to read the PORT # from the system environment. Therefore, you will need diff --git a/apps/site/config/test.exs b/apps/site/config/test.exs index 2320e84a6c..4c43bc571d 100644 --- a/apps/site/config/test.exs +++ b/apps/site/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # We don't run a server during test. If one is required, # you can enable the server option below. @@ -13,8 +13,7 @@ config :site, :secure_pipeline, rewrite_on: [:x_forwarded_proto] ] -# Print only warnings and errors during test -config :logger, level: :warn +config :logger, level: :notice # Don't fetch tz data in test mode; can cause a race if we're doing TZ # operations while it updates. diff --git a/apps/site/lib/site_web/controllers/cms_controller.ex b/apps/site/lib/site_web/controllers/cms_controller.ex index c458131a20..f835ace526 100644 --- a/apps/site/lib/site_web/controllers/cms_controller.ex +++ b/apps/site/lib/site_web/controllers/cms_controller.ex @@ -43,6 +43,34 @@ defmodule SiteWeb.CMSController do |> handle_page_response(conn) end + def reset_cache_key(conn, %{"object" => object, "id" => id}) do + Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}") + + try do + CMS.Cache.delete("/cms/#{object}/#{id}") + + Logger.notice("cms.cache.delete path=/cms/#{object}/#{id}") + rescue + e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}") + end + + send_resp(conn, 202, "") |> halt() + end + + def reset_cache_key(conn, %{"id" => id}) do + Logger.notice("cms.cache.delete redis_host=#{System.get_env("REDIS_HOST")}") + + try do + CMS.Cache.delete("/cms/#{id}") + + Logger.notice("cms.cache.delete path=/cms/#{id}") + rescue + e in Redix.ConnectionError -> Logger.warning("cms.cache.delete error=redis-#{e.reason}") + end + + send_resp(conn, 202, "") |> halt() + end + @spec handle_page_response(Page.t() | {:error, API.error()}, Conn.t()) :: Plug.Conn.t() defp handle_page_response(%{__struct__: struct} = page, conn) diff --git a/apps/site/lib/site_web/router.ex b/apps/site/lib/site_web/router.ex index ac80077087..711313e157 100644 --- a/apps/site/lib/site_web/router.ex +++ b/apps/site/lib/site_web/router.ex @@ -51,6 +51,13 @@ defmodule SiteWeb.Router do get("/_health", HealthController, :index) end + scope "/cms", SiteWeb do + pipe_through([:basic_auth]) + + patch("/:object/:id", CMSController, :reset_cache_key) + patch("/:id", CMSController, :reset_cache_key) + end + # redirect 't.mbta.com' and 'beta.mbta.com' to 'https://www.mbta.com' scope "/", SiteWeb, host: "t." do # no pipe @@ -264,6 +271,12 @@ defmodule SiteWeb.Router do get("/*path", CMSController, :page) end + defp basic_auth(conn, _) do + opts = Application.get_env(:site, SiteWeb.Router)[:cms_basic_auth] + + Plug.BasicAuth.basic_auth(conn, opts) + end + defp optional_disable_indexing(conn, _) do if Application.get_env(:site, :allow_indexing) do conn diff --git a/apps/site/test/site_web/controllers/cms_controller_test.exs b/apps/site/test/site_web/controllers/cms_controller_test.exs index 5dcc624308..430e285e69 100644 --- a/apps/site/test/site_web/controllers/cms_controller_test.exs +++ b/apps/site/test/site_web/controllers/cms_controller_test.exs @@ -1,6 +1,8 @@ defmodule SiteWeb.CMSControllerTest do use SiteWeb.ConnCase, async: false + import ExUnit.CaptureLog + alias Plug.Conn describe "GET - page" do @@ -182,4 +184,15 @@ defmodule SiteWeb.CMSControllerTest do assert html_response(conn, 500) =~ "Something went wrong on our end." end end + + describe "PATCH /cms/*" do + test "it logs that no redis connection was made", %{conn: conn} do + assert capture_log(fn -> + conn + |> put_req_header("content-type", "application/json") + |> put_req_header("authorization", "Basic " <> Base.encode64("username:password")) + |> patch("/cms/foo/bar") + end) =~ "cms.cache.delete error=redis-closed" + end + end end diff --git a/config/runtime.exs b/config/runtime.exs new file mode 100644 index 0000000000..0187116e8f --- /dev/null +++ b/config/runtime.exs @@ -0,0 +1,27 @@ +import Config + +config :cms, CMS.Cache, + conn_opts: [ + host: System.get_env("REDIS_HOST", "127.0.0.1"), + port: 6379 + ], + stats: false, + telemetry: false + +if config_env() == :test do + config :site, SiteWeb.Router, + cms_basic_auth: [ + username: "username", + password: "password" + ] +else + config :site, SiteWeb.Router, + cms_basic_auth: [ + username: System.fetch_env!("CMS_BASIC_AUTH_USERNAME"), + password: System.fetch_env!("CMS_BASIC_AUTH_PASSWORD") + ] +end + +if config_env() == :prod do + config :alerts, bus_stop_change_bucket: System.get_env("S3_PREFIX_BUSCHANGE") +end diff --git a/mix.lock b/mix.lock index 614f23623b..271b04cc98 100644 --- a/mix.lock +++ b/mix.lock @@ -56,6 +56,8 @@ "mint_web_socket": {:hex, :mint_web_socket, "1.0.3", "aab42fff792a74649916236d0b01f560a0b3f03ca5dea693c230d1c44736b50e", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "ca3810ca44cc8532e3dce499cc17f958596695d226bb578b2fbb88c09b5954b0"}, "mochiweb": {:hex, :mochiweb, "2.22.0", "f104d6747c01a330c38613561977e565b788b9170055c5241ac9dd6e4617cba5", [:rebar3], [], "hexpm", "cbbd1fd315d283c576d1c8a13e0738f6dafb63dc840611249608697502a07655"}, "mock": {:hex, :mock, "0.3.3", "42a433794b1291a9cf1525c6d26b38e039e0d3a360732b5e467bfc77ef26c914", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "a280d1f7b6f4bbcbd9282616e57502721781c66ee5b540720efabeaf627cc7eb"}, + "nebulex": {:hex, :nebulex, "2.5.2", "2d358813ccb2eeea525e3a29c270ad123d3337e97ed9159d9113cf128108bd4c", [:mix], [{:decorator, "~> 1.4", [hex: :decorator, repo: "hexpm", optional: true]}, {:shards, "~> 1.1", [hex: :shards, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "61a122302cf42fa61eca22515b1df21aaaa1b98cf462f6dd0998de9797aaf1c7"}, + "nebulex_redis_adapter": {:hex, :nebulex_redis_adapter, "2.3.1", "ea57629ee21f78332ca8d0356e6777d2fdbd6755b7453e298557091b6f8811f6", [:mix], [{:crc, "~> 0.10", [hex: :crc, repo: "hexpm", optional: true]}, {:jchash, "~> 0.1", [hex: :jchash, repo: "hexpm", optional: true]}, {:nebulex, "~> 2.5", [hex: :nebulex, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.5 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:redix, "~> 1.2", [hex: :redix, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "8384bf249af7c0b9903578b5c75a18157562863054952dbaea55dfe7255b75e7"}, "nimble_options": {:hex, :nimble_options, "1.0.2", "92098a74df0072ff37d0c12ace58574d26880e522c22801437151a159392270e", [:mix], [], "hexpm", "fd12a8db2021036ce12a309f26f564ec367373265b53e25403f0ee697380f1b8"}, "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, "nimble_pool": {:hex, :nimble_pool, "1.0.0", "5eb82705d138f4dd4423f69ceb19ac667b3b492ae570c9f5c900bb3d2f50a847", [:mix], [], "hexpm", "80be3b882d2d351882256087078e1b1952a28bf98d0a287be87e4a24a710b67a"}, @@ -81,6 +83,7 @@ "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recaptcha": {:git, "https://github.com/samueljseay/recaptcha.git", "8ea13f63990ca18725ac006d30e55d42c3a58457", [ref: "8ea13f63990ca18725ac006d30e55d42c3a58457"]}, "recon": {:hex, :recon, "2.5.1", "430ffa60685ac1efdfb1fe4c97b8767c92d0d92e6e7c3e8621559ba77598678a", [:mix, :rebar3], [], "hexpm", "5721c6b6d50122d8f68cccac712caa1231f97894bab779eff5ff0f886cb44648"}, + "redix": {:hex, :redix, "1.3.0", "f4121163ff9d73bf72157539ff23b13e38422284520bb58c05e014b19d6f0577", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "60d483d320c77329c8cbd3df73007e51b23f3fae75b7693bc31120d83ab26131"}, "req": {:hex, :req, "0.3.12", "f84c2f9e7cc71c81d7cbeacf7c61e763e53ab5f3065703792a4ab264b4f22672", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "c91103d4d1c8edeba90c84e0ba223a59865b673eaab217bfd17da3aa54ab136c"}, "rstar": {:git, "https://github.com/armon/erl-rstar.git", "a406b2cce609029bf65b9ccfbe93a0416c0ee0cd", []}, "sentry": {:hex, :sentry, "7.0.4", "a9a00b480becfca5b897a1b383f88e21fa21abc876e2b1b30060040f76a9c776", [:mix], [{:hackney, "~> 1.8 or 1.6.5", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "0aa7e405388af5634aefad7f3e11f188e8c13d0003ae7100b956eb01c8b002a5"}, diff --git a/rel/config.exs b/rel/config.exs index c22b5db203..8ccf385bcb 100644 --- a/rel/config.exs +++ b/rel/config.exs @@ -24,7 +24,7 @@ environment :prod do set( overlays: [ - {:copy, "apps/alerts/config/runtime.exs", "etc/runtime.exs"} + {:copy, "config/runtime.exs", "etc/runtime.exs"} ] )