Use local httpbin to run tests

Fix #132 Start a local httpbin container to run tests against. Https requests are still using remote one since it's hard to setup local secure ssl. The requests using https are simple and during my tests I haven't seen any timeout.
com-lihaoyi · Sep 15, 2024 · 374c175 · 374c175
1 parent beb5d26
commit 374c175
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 45 deletions.
diff --git a/build.sc b/build.sc
@@ -40,7 +40,8 @@ trait RequestsModule extends CrossScalaModule with PublishModule with Mima {
   object test extends ScalaTests with TestModule.Utest {
     def ivyDeps = Agg(
       ivy"com.lihaoyi::utest::0.7.10",
-      ivy"com.lihaoyi::ujson::1.3.13"
+      ivy"com.lihaoyi::ujson::1.3.13",
+      ivy"com.dimafeng::testcontainers-scala-core:0.41.3"
     )
   }
 }
diff --git a/requests/test/src/requests/RequestTests.scala b/requests/test/src/requests/RequestTests.scala
@@ -1,9 +1,25 @@
 package requests
 
+import com.dimafeng.testcontainers.GenericContainer
+import org.testcontainers.containers.wait.strategy.Wait
 import utest._
 import ujson._
 
 object RequestTests extends TestSuite{
+
+  val containerDef = GenericContainer.Def(
+    "kennethreitz/httpbin",
+    exposedPorts = Seq(80),
+    waitStrategy = Wait.forHttp("/")
+  )
+  val container = containerDef.start()
+  val localHttpbinHost = container.containerIpAddress
+  val localHttpbin = s"${localHttpbinHost}:${container.mappedPort(80)}"
+
+  override def utestAfterAll(): Unit = {
+    container.stop()
+  }
+
   val tests = Tests{
     test("matchingMethodWorks"){
       val requesters = Seq(
@@ -13,18 +29,18 @@ object RequestTests extends TestSuite{
         requests.put
       )
 
-      for(protocol <- Seq("http", "https")){
+      for(baseUrl <- Seq(s"http://$localHttpbin", "https://httpbin.org")){
         for(r <- requesters){
           for(r2 <- requesters){
-            val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}", check = false)
+            val res = r(s"$baseUrl/${r2.verb.toLowerCase()}", check = false)
             if (r.verb == r2.verb) assert(res.statusCode == 200)
             else assert(res.statusCode == 405)
 
             if (r.verb == r2.verb){
-              val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              val res = r(s"$baseUrl/${r2.verb.toLowerCase()}")
               assert(res.statusCode == 200)
             }else intercept[RequestFailedException]{
-              r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              r(s"$baseUrl/${r2.verb.toLowerCase()}")
             }
           }
         }
@@ -34,26 +50,26 @@ object RequestTests extends TestSuite{
     test("params"){
       test("get"){
         // All in URL
-        val res1 = requests.get("https://httpbin.org/get?hello=world&foo=baz").text()
+        val res1 = requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text()
         assert(read(res1).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // All in params
         val res2 = requests.get(
-          "https://httpbin.org/get",
+          s"http://$localHttpbin/get",
           params = Map("hello" -> "world", "foo" -> "baz")
         )
         assert(read(res2).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Mixed URL and params
         val res3 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("foo" -> "baz")
         ).text()
         assert(read(res3).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Needs escaping
         val res4 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("++-- lol" -> " !@#$%")
         )
         assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world"))
@@ -63,7 +79,7 @@ object RequestTests extends TestSuite{
     test("multipart"){
       for(chunkedUpload <- Seq(true, false)) {
         val response = requests.post(
-          "http://httpbin.org/post",
+          s"http://$localHttpbin/post",
           data = MultiPart(
             MultiItem("file1", "Hello!".getBytes, "foo.txt"),
             MultiItem("file2", "Goodbye!")
@@ -79,76 +95,77 @@ object RequestTests extends TestSuite{
     test("cookies"){
       test("session"){
         val s = requests.Session(cookieValues = Map("hello" -> "world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test", "hello" -> "world")))
       }
       test("raw"){
-        val res1 = requests.get("https://httpbin.org/cookies").text().trim
+        val res1 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj()))
-        requests.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = requests.get("https://httpbin.org/cookies").text().trim
+        requests.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj()))
       }
       test("space"){
         val s = requests.Session(cookieValues = Map("hello" -> "hello, world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "hello, world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test+test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test+test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")))
       }
     }
 
     test("redirects"){
       test("max"){
-        val res1 = requests.get("https://httpbin.org/absolute-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/absolute-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/absolute-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/absolute-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/absolute-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/absolute-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
       test("maxRelative"){
-        val res1 = requests.get("https://httpbin.org/relative-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/relative-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/relative-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/relative-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/relative-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/relative-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
     }
 
     test("test_reproduction"){
-      requests.get("http://httpbin.org/status/304").text()
+      requests.get(s"http://$localHttpbin/status/304").text()
 
     }
     test("streaming"){
-      val res1 = requests.get("http://httpbin.org/stream/5").text()
+      val res1 = requests.get(s"http://$localHttpbin/stream/5").text()
       assert(res1.linesIterator.length == 5)
-      val res2 = requests.get("http://httpbin.org/stream/52").text()
+      val res2 = requests.get(s"http://$localHttpbin/stream/52").text()
       assert(res2.linesIterator.length == 52)
     }
 
     test("timeouts"){
       test("read"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", readTimeout = 10)
+          requests.get(s"http://$localHttpbin/delay/1", readTimeout = 10)
         }
-        requests.get("https://httpbin.org/delay/1", readTimeout = 2000)
+        requests.get(s"http://$localHttpbin/delay/1", readTimeout = 2000)
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/3", readTimeout = 2000)
+          requests.get(s"http://$localHttpbin/delay/3", readTimeout = 2000)
         }
       }
       test("connect"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", connectTimeout = 1)
+          // use remote httpbin.org so it needs more time to connect
+          requests.get(s"https://httpbin.org/delay/1", connectTimeout = 1)
         }
       }
     }
@@ -167,38 +184,38 @@ object RequestTests extends TestSuite{
     }
 
     test("decompress"){
-      val res1 = requests.get("https://httpbin.org/gzip")
-      assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org")
+      val res1 = requests.get(s"http://$localHttpbin/gzip")
+      assert(read(res1.text()).obj("headers").obj("Host").str == localHttpbin)
 
-      val res2 = requests.get("https://httpbin.org/deflate")
-      assert(read(res2).obj("headers").obj("Host").str == "httpbin.org")
+      val res2 = requests.get(s"http://$localHttpbin/deflate")
+      assert(read(res2).obj("headers").obj("Host").str == localHttpbin)
 
-      val res3 = requests.get("https://httpbin.org/gzip", autoDecompress = false)
+      val res3 = requests.get(s"http://$localHttpbin/gzip", autoDecompress = false)
       assert(res3.bytes.length < res1.bytes.length)
 
-      val res4 = requests.get("https://httpbin.org/deflate", autoDecompress = false)
+      val res4 = requests.get(s"http://$localHttpbin/deflate", autoDecompress = false)
       assert(res4.bytes.length < res2.bytes.length)
 
       (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length)
     }
 
     test("compression"){
       val res1 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.None,
         data = new RequestBlob.ByteSourceRequestBlob("Hello World")
       )
       assert(res1.text().contains(""""Hello World""""))
 
       val res2 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Gzip,
         data = new RequestBlob.ByteSourceRequestBlob("I am cow")
       )
       assert(read(new String(res2.bytes))("data").toString.contains("data:application/octet-stream;base64,H4sIAAAAAA"))
 
       val res3 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Deflate,
         data = new RequestBlob.ByteSourceRequestBlob("Hear me moo")
       )
@@ -208,7 +225,7 @@ object RequestTests extends TestSuite{
 
     test("headers"){
       test("default"){
-        val res = requests.get("https://httpbin.org/headers").text()
+        val res = requests.get(s"http://$localHttpbin/headers").text()
         val hs = read(res)("headers").obj
         assert(hs("User-Agent").str == "requests-scala")
         assert(hs("Accept-Encoding").str == "gzip, deflate")
@@ -306,7 +323,7 @@ object RequestTests extends TestSuite{
     // to the server. This preserves the 0.8.x behavior, and can always be overriden
     // by passing a comma-separated list of headers instead
     test("duplicateHeaders"){
-      val res = requests.get("https://httpbin.org/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
+      val res = requests.get(s"http://$localHttpbin/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
       assert(ujson.read(res)("headers")("X-Y") == Str("b")) // make sure it's not "a,b"
     }
   }