diff --git a/src/main/scala/com/aivean/royalroad/Args.scala b/src/main/scala/com/aivean/royalroad/Args.scala index 9793e34..b14f106 100644 --- a/src/main/scala/com/aivean/royalroad/Args.scala +++ b/src/main/scala/com/aivean/royalroad/Args.scala @@ -30,6 +30,12 @@ class Args(args: Seq[String]) extends ScallopConf(args) { default = Some(false) ) + val includeTitlePage = opt[Boolean]( + descr = "Include title page with fiction title and author name", + noshort = true, + default = Some(false) + ) + val removeWarnings = opt[Boolean]( descr = "Remove warnings about reporting story if found on Amazon", noshort = true, diff --git a/src/main/scala/com/aivean/royalroad/Main.scala b/src/main/scala/com/aivean/royalroad/Main.scala index c83306c..ebbfae5 100644 --- a/src/main/scala/com/aivean/royalroad/Main.scala +++ b/src/main/scala/com/aivean/royalroad/Main.scala @@ -17,6 +17,7 @@ import scala.util.{Failure, Success, Try} object Main extends App { print ("Royalroad downloader v" + VersionInfo.version + "\n") + val cliArgs = new Args(args) def handleFromArg[T](chaps: Seq[T], fromChap: Int): Seq[T] = if (fromChap > 0) chaps.drop(fromChap - 1) else if (fromChap < 0) chaps.takeRight(-fromChap) else chaps @@ -29,7 +30,10 @@ object Main extends App { } } - val cliArgs = new Args(args) + def embedImageIfNeeded(url: String): Try[String] = + if (cliArgs.embedImages()) { + Try(new URL(url)).flatMap(url => Try(retry(getDataURIForURL(url)))).map(_.toString) + } else Success(url) import DSL.Extract._ import DSL._ @@ -97,6 +101,48 @@ object Main extends App { try { printWriter.write(s"""$title""") + if (cliArgs.includeTitlePage()) try { + // include header with title, author, description and cover image + val authorName = doc >> text("h4 > span > a") + val authorProfilePic = doc >> attr("src")("div.avatar-container-general > img") + val fictionDescription = doc >> text("div.description") + val fictionImage = doc >> attr("src")("div.cover-art-container > img") + val fictionLink = extractFictionLink(cliArgs.fictionLink()) // get the fiction page link + // get the current date and time + + println("Author: " + authorName) + println("Author Profile Picture: " + authorProfilePic) + println("Description: " + fictionDescription) + println("Image: " + fictionImage) + println("Fiction Link: " + fictionLink) + + def embedIfNeededSilently(url: String) = embedImageIfNeeded(url).getOrElse(url) + + printWriter.write( +
+

+ + {title} + +

+ +

by + {authorName} +

+

+ {fictionDescription} +

+

Scraped at: + {new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new java.util.Date())} +

+
.toString() + ) + } catch { + case e: Exception => + println("Failed to include title page: " + e.getMessage) + e.printStackTrace() + } + // process the queue until the end message None is received Stream.continually(chapQ.take()) .takeWhile(_.isDefined) @@ -123,15 +169,12 @@ object Main extends App { case img: JsoupElement if img.hasAttr("src") => val imgUrl = img.attr("src") println("embedding image: " + imgUrl) - Try(new URL(imgUrl)) match { - case Success(url) => - Try(retry(getDataURIForURL(url))) match { - case Success(dataUrl) => img.underlying.attr("src", dataUrl.toString) - case Failure(e) => - println(s"Failed to convert $imgUrl to data URL") - e.printStackTrace() - } - case Failure(_) => println(s"Invalid URL: $imgUrl") + + embedImageIfNeeded(imgUrl) match { + case Success(dataUrl) => img.underlying.attr("src", dataUrl) + case Failure(e) => + println(s"Failed to convert $imgUrl to data URL") + e.printStackTrace() } case img: JsoupElement => println(s"Warning: image without src attribute: ${img.outerHtml} in $url") } diff --git a/src/main/scala/com/aivean/royalroad/Utils.scala b/src/main/scala/com/aivean/royalroad/Utils.scala index def89ae..c1482f4 100644 --- a/src/main/scala/com/aivean/royalroad/Utils.scala +++ b/src/main/scala/com/aivean/royalroad/Utils.scala @@ -4,6 +4,8 @@ import java.io.{BufferedInputStream, ByteArrayOutputStream, IOException} import java.net._ import java.util.Base64 import java.util.concurrent.atomic.AtomicLong +import javax.net.ssl.SSLContext +import javax.security.cert.X509Certificate import scala.util.{Failure, Success, Try} @@ -157,4 +159,23 @@ object Utils { dir.mkdirs() } } + + def disableSSL(): Unit = { + import java.security.cert.X509Certificate + import javax.net.ssl.{SSLContext, TrustManager, X509TrustManager} + import javax.net.ssl.HttpsURLConnection + + val trustAllCerts: Array[TrustManager] = Array[TrustManager](new X509TrustManager() { + def getAcceptedIssuers: Array[X509Certificate] = null + + def checkClientTrusted(certs: Array[X509Certificate], authType: String): Unit = {} + + def checkServerTrusted(certs: Array[X509Certificate], authType: String): Unit = {} + }) + + val sc: SSLContext = SSLContext.getInstance("SSL") + sc.init(null, trustAllCerts, new java.security.SecureRandom) + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory) + } + } diff --git a/src/test/scala/com/aivean/royalroad/UtilsTest.scala b/src/test/scala/com/aivean/royalroad/UtilsTest.scala index 2b6f387..04fa026 100644 --- a/src/test/scala/com/aivean/royalroad/UtilsTest.scala +++ b/src/test/scala/com/aivean/royalroad/UtilsTest.scala @@ -16,6 +16,14 @@ class UtilsTest extends FunSuite { } + test("urlToDataUri2") { + // small red dot + val url = new URL("https://www.royalroadcdn.com/public/avatars/avatar-12400-AADA0m2ygBQ.png") + val dataUri = Utils.getDataURIForURL(url) + println(dataUri) + } + + val reportWarnings = List( "A case of theft: this story is not rightfully on Amazon; if you spot it, report the violation.", "The narrative has been stolen; if detected on Amazon, report the infringement.",