Skip to content

Commit

Permalink
introduce negative --from values to download last chapters
Browse files Browse the repository at this point in the history
more tests
  • Loading branch information
Aivean committed Aug 21, 2023
1 parent c95b524 commit f4a290e
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 6 deletions.
5 changes: 4 additions & 1 deletion readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,11 @@ How to use
* download the latest [release .jar](https://github.com/Aivean/royalroad-downloader/releases/latest)
* install the latest java (maybe you have one, check by executing `java -version`)
* execute `java -jar royalroad-downloader-assembly-VERSION.jar https://royalroadl.com/fiction/...`
(replace royalroad fiction URL with your link and `VERSION` with the corresponding number)
(replace royalroad fiction URL with your link and `VERSION` with the corresponding number)
* the output html file will be created in the current directory


Run `java -jar royalroad-downloader-assembly-VERSION.jar --help` to see the full usage reference.


Troubleshooting
Expand Down
9 changes: 6 additions & 3 deletions src/main/scala/com/aivean/royalroad/Args.scala
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@ class Args(args: Seq[String]) extends ScallopConf(args) {
| """.stripMargin)

val fromChapter = opt[Int](
descr = "Start download from chapter #",
default = Some(1),
validate = _ >= 1
descr = "Start download from chapter #. " +
"Positive values indicate chapters from the beginning. " +
"(1 means start from the first chapter, 2 means second chapter) " +
"Negative values indicate chapters from the end " +
"(-1 means last chapter, -2 means two chapters from the end)",
default = Some(1)
)

val titleQuery = opt[String](
Expand Down
23 changes: 21 additions & 2 deletions src/main/scala/com/aivean/royalroad/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ import scala.concurrent.{Await, Future, duration}

object Main extends App {

def handleFromArg[T](chaps: Seq[T], fromChap: Int): Seq[T] =
if (fromChap > 0) chaps.drop(fromChap - 1) else if (fromChap < 0) chaps.takeRight(-fromChap) else chaps


val cliArgs = new Args(args)

import DSL.Extract._
Expand All @@ -40,20 +44,34 @@ object Main extends App {
case x => x
}

val chapUrlsConstrained = handleFromArg(chapUrls, cliArgs.fromChapter())
if (chapUrlsConstrained.isEmpty) {
println("No chapters found")
System.exit(1)
}

import scala.concurrent.ExecutionContext.Implicits.global

// chapter producer, with parallelism limited by the capacity of queue (currently capacity = 4)
val chapQ = new ArrayBlockingQueue[Option[Future[(String, Document)]]](4, true)
Future {
chapUrls.drop(cliArgs.fromChapter() - 1).foreach { u =>
chapUrlsConstrained.foreach { u =>
val uDecoded = URLDecoder.decode(u, "utf-8")
println(s"downloading: $uDecoded")
chapQ.put(Some(Future(uDecoded -> retry(backpressure(browser.get(uDecoded))))))
}
chapQ.put(None)
}

val filename = title.replaceAll("[^\\w\\d]+", "_") + ".html"
val filename = title.replaceAll("[^\\w\\d]+", "_") + {
// when chapter range is specified, add it to the filename
if (chapUrls.size != chapUrlsConstrained.size) {
val firstChapter = chapUrls.indexOf(chapUrlsConstrained.head)
val lastChapter = chapUrls.indexOf(chapUrlsConstrained.last)
if (firstChapter == lastChapter) "_chapter_" + (firstChapter + 1) else
"_chapters_" + (firstChapter + 1) + "-" + (lastChapter + 1)
} else ""
} + ".html"
println("Saving as: " + filename)

val printWriter = new PrintWriter(filename, "UTF-8")
Expand Down Expand Up @@ -90,4 +108,5 @@ object Main extends App {
}

println("done")
println("Saved: " + filename)
}
38 changes: 38 additions & 0 deletions src/test/scala/com/aivean/royalroad/MainTests.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package com.aivean.royalroad

import com.aivean.royalroad.Main.handleFromArg
import org.scalatest.FunSuite

class MainTests extends FunSuite {

test("handleFromArg should drop elements when fromChap is positive") {
val chaps = Seq(1, 2, 3, 4, 5)
val result = handleFromArg(chaps, 3)
assert(result == Seq(3, 4, 5))
}

test("handleFromArg should take right elements when fromChap is negative") {
val chaps = Seq(1, 2, 3, 4, 5)
val result = handleFromArg(chaps, -2)
assert(result == Seq(4, 5))
}

test("handleFromArg should return the same sequence when fromChap is zero") {
val chaps = Seq(1, 2, 3, 4, 5)
val result = handleFromArg(chaps, 0)
assert(result == chaps)
}

test("handleFromArg should handle an empty sequence") {
val chaps = Seq.empty[Int]
val result = handleFromArg(chaps, 2)
assert(result.isEmpty)
}

// If you want to go further and test with different types
test("handleFromArg should handle sequence of Strings") {
val chaps = Seq("A", "B", "C")
val result = handleFromArg(chaps, -1)
assert(result == Seq("C"))
}
}

0 comments on commit f4a290e

Please sign in to comment.