Merge pull request #19 from sekiju/v1.1.0

V1.1.0

CHANGELOG.md

@@ -19,13 +19,38 @@ Date: YYYY-MM-DD
 
 -->
 
-## v1.0.1
+## v1.1.0
 
 Date: YYYY-MM-DD
 
+### What's Changed
+
+### Configuration Changes
+
+All variables have been renamed. For a detailed overview of the changes, please refer to the [example.config.hcl](https://github.com/sekiju/mdl/blob/c6bcfaf5ce28b6b73abebb6a6db97e25803f9f1e/example.config.hcl) file. Note that only the section names remain unchanged; all other variables have been updated.
+
+- The `download` section has been removed.
+  - The variable `concurrent_downloads` has been renamed to `max_parallel_downloads` and moved to the `application` section.
+
+- `check_for_updates` has been renamed to `check_updates`.
+- `dir` has been renamed to `directory`.
+- `clean_dir` has been renamed to `clean_on_start`.
+- `format` has been renamed to `file_format`.
+- In the `site` section, `cookie_string` has been renamed to `cookie`.
+
+#### Support for new Websites
+
+- [www.corocoro.jp](https://www.corocoro.jp/) ([#4f4d590](https://github.com/sekiju/mdl/commit/4f4d590d606371455b803af38007edbeec047fad))
+- [storia.takeshobo.co.jp](https://storia.takeshobo.co.jp/) ([#5205970](https://github.com/sekiju/mdl/commit/520597093fb45e00602c78c78e34829df4d43284))
+
+## v1.0.1
+
+Date: 2024-11-23
+
 ### Patch Changes
 
-- Fix the `unsupported provider for hostname:` error when the user launches the app from Explorer ([#5ba31cc](https://github.com/sekiju/mdl/commit/5ba31cc023d1abb9f92adfacb8319d2310ae2760)).
+- Fix the `unsupported provider for hostname:` error when the user launches the app from
+  Explorer ([#5ba31cc](https://github.com/sekiju/mdl/commit/5ba31cc023d1abb9f92adfacb8319d2310ae2760)).
 
 ## v1.0.0
 
@@ -37,8 +62,8 @@ Date: 2024-11-22
 - **Configuration file format updated:** Replaced the YAML config with a new HCL (HashiCorp Configuration Language) format
 - **Rewritten internal API for extensions:** The internal API has been refactored to improve extensibility and maintainability
 - **New command-line arguments:**
-  - `--cookie (string)`: Provides the cookie string for the current session
-  - `--config (string)`: Specifies the path to the configuration file (default: `config.hcl`)
+    - `--cookie (string)`: Provides the cookie string for the current session
+    - `--config (string)`: Specifies the path to the configuration file (default: `config.hcl`)
 - **Support for multi-download:** Now you can pass multiple links, and all of them will be downloaded
 - **Better performance:** Optimizations across the system have resulted in a +56% performance boost
 

README.md

@@ -44,7 +44,7 @@ From this, the optimal number of threads is approximately **13**.
 #### Why not always use maximum threads?
 While increasing threads can improve download speeds, setting too high a number, like `50`, places unnecessary strain on your computer’s resources, including disk, processor, and memory usage. This extra load does not significantly improve performance and may degrade overall system efficiency. Therefore, it’s advisable to determine and use an optimal thread count for your setup.
 
-### Paid chapter download or how to obtain `cookie_string`?
+### Paid chapter download or how to obtain `cookie`?
 
 To extract cookies from websites, use tools like [Cookie-Editor](https://cookie-editor.com).
 
@@ -64,7 +64,7 @@ CLI argument or modify config file:
 
 site {
   "shonenjumpplus.com" {
-    cookie_string = "glsc=1hYa4GrNp2DndSNIShVyoDGP6MgDmaJhiX22C0X734hkzod56wsBN7Fy1S5ZBOQd"
+    cookie = "glsc=1hYa4GrNp2DndSNIShVyoDGP6MgDmaJhiX22C0X734hkzod56wsBN7Fy1S5ZBOQd"
   }
 }
 ```

bench/.gitignore

@@ -0,0 +1,2 @@
+episode/
+public/

bench/generate_benchmark_server.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gofiber/fiber/v2"
+	"github.com/rs/zerolog/log"
+	"github.com/sekiju/htt"
+	"github.com/sekiju/mdl/extractor/util"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	chapterURL = "https://comic-ogyaaa.com/episode/2550912964910100594"
+)
+
+/* update hosts
+127.0.0.1 comic-ogyaaa.com
+127.0.0.1 cdn-img.comic-ogyaaa.com
+*/
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal().Err(err).Send()
+	}
+}
+
+func run() error {
+	res, err := htt.New().SetHeader("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 17_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Mobile/15E148 Safari/604.1").Get(chapterURL)
+	if err != nil {
+		return err
+	}
+
+	html, err := res.Text()
+	if err != nil {
+		return err
+	}
+
+	episode, err := util.ExtractJSONFromHTML[episodeResult](html, `<script id='episode-json' type='text/json' data-value='`, `'></script>`)
+	if err != nil {
+		return err
+	}
+
+	for _, page := range episode.ReadableProduct.PageStructure.Pages {
+		if page.Type != "main" {
+			continue
+		}
+
+		parsedURL, _ := url.Parse(page.Src)
+		filePath := filepath.Join(".", parsedURL.Path)
+
+		if _, err = os.Stat(filePath); err == nil {
+			continue
+		}
+
+		if err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+			return err
+		}
+
+		res, err = htt.New().Get(page.Src)
+		if err != nil {
+			return err
+		}
+
+		file, err := os.Create(filePath)
+		if err != nil {
+			return fmt.Errorf("failed to create file: %w", err)
+		}
+
+		_, err = file.ReadFrom(res.Body)
+		if err != nil {
+			return fmt.Errorf("failed to write file: %w", err)
+		}
+
+		if err = file.Close(); err != nil {
+			return err
+		}
+	}
+
+	parsedURL, _ := url.Parse(chapterURL)
+	filePath := filepath.Join(".", parsedURL.Path)
+
+	if err = os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
+		return err
+	}
+
+	html = strings.ReplaceAll(html, "https://", "http://")
+
+	if err = os.WriteFile(filePath, []byte(html), os.ModePerm); err != nil {
+		return err
+	}
+
+	app := fiber.New()
+
+	app.Static("/", ".")
+
+	return app.Listen(":80")
+}
+
+type episodeResult struct {
+	ReadableProduct struct {
+		FinishReadingNotificationUri interface{} `json:"finishReadingNotificationUri"`
+		HasPurchased                 bool        `json:"hasPurchased"`
+		Id                           string      `json:"id"`
+		IsPublic                     bool        `json:"isPublic"`
+		NextReadableProductUri       *string     `json:"nextReadableProductUri"`
+		Number                       int         `json:"number"`
+		PageStructure                *struct {
+			Pages []episodeResultPage `json:"pages"`
+		} `json:"pageStructure"`
+		Permalink              string  `json:"permalink"`
+		PrevReadableProductUri *string `json:"prevReadableProductUri"`
+		Title                  string  `json:"title"`
+	} `json:"readableProduct"`
+}
+
+type episodeResultPage struct {
+	Type string `json:"type"`
+	Src  string `json:"src,omitempty"`
+}

bench/go.mod

@@ -0,0 +1,31 @@
+module bench_server
+
+go 1.23
+
+require (
+	github.com/gofiber/fiber/v2 v2.52.5
+	github.com/rs/zerolog v1.33.0
+	github.com/sekiju/htt v0.1.1
+	github.com/sekiju/mdl v0.0.0
+)
+
+replace github.com/sekiju/mdl v0.0.0 => ..
+
+require (
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/klauspost/compress v1.17.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/valyala/tcplisten v1.0.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

bench/main_test.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"github.com/sekiju/mdl/downloader"
+	"testing"
+)
+
+func Benchmark(b *testing.B) {
+	chapterURLs := []string{"http://comic-ogyaaa.com/episode/2550912964910100594"}
+
+	b.Run("Raw", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			loader := downloader.NewDownloader()
+
+			for _, chapterURL := range chapterURLs {
+				loader.Queue(chapterURL)
+			}
+
+			loader.Stop()
+		}
+	})
+}