From daedd130a31a3790449e676aa5df204d9de703bf Mon Sep 17 00:00:00 2001 From: shavit Date: Fri, 9 Feb 2024 11:51:44 -0500 Subject: [PATCH 1/2] Bilibili festival This change will work with festival pages. --- extractors/bilibili/bilibili.go | 34 ++++++++++++++++++++++++++-- extractors/bilibili/bilibili_test.go | 7 ++++++ extractors/bilibili/types.go | 27 ++++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index ca8cf5edb..f0da13ef1 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -202,6 +202,33 @@ func getMultiPageData(html string) (*multiPage, error) { return &data, nil } +func extractFestival(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { + matches := utils.MatchAll(html, "<\\s*script[^>]*>\\s*window\\.__INITIAL_STATE__=([\\s\\S]*?);\\s?\\(function[\\s\\S]*?<\\/\\s*script\\s*>") + if len(matches) < 1 { + return nil, errors.WithStack(extractors.ErrURLParseFailed) + } + if len(matches[0]) < 2 { + return nil, errors.New("could not find video in page") + } + + var festivalData festival + err := json.Unmarshal([]byte(matches[0][1]), &festivalData) + if err != nil { + return nil, errors.WithStack(err) + } + + options := bilibiliOptions{ + url: url, + html: html, + aid: festivalData.VideoInfo.Aid, + bvid: festivalData.VideoInfo.BVid, + cid: festivalData.VideoInfo.Cid, + page: 0, + } + + return []*extractors.Data{bilibiliDownload(options, extractOption)}, nil +} + func extractNormalVideo(url, html string, extractOption extractors.Options) ([]*extractors.Data, error) { pageData, err := getMultiPageData(html) if err != nil { @@ -333,9 +360,12 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor if strings.Contains(url, "bangumi") { // handle bangumi return extractBangumi(url, html, option) + } else if strings.Contains(url, "festival") { + return extractFestival(url, html, option) + } else { + // handle normal video + return extractNormalVideo(url, html, option) } - // handle normal video - return extractNormalVideo(url, html, option) } // bilibiliDownload is the download function for a single URL diff --git a/extractors/bilibili/bilibili_test.go b/extractors/bilibili/bilibili_test.go index 471861f74..f1ad0ecbc 100644 --- a/extractors/bilibili/bilibili_test.go +++ b/extractors/bilibili/bilibili_test.go @@ -66,6 +66,13 @@ func TestBilibili(t *testing.T) { Title: "【十年榜】2000-2009年最强华语金曲TOP100 P1 100爱转角-罗志祥", }, }, + { + name: "festival test", + args: test.Args{ + URL: "https://www.bilibili.com/festival/lty10th?bvid=BV1dZ4y1Y7bt", + Title: "洛天依十周年官方演唱会", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/extractors/bilibili/types.go b/extractors/bilibili/types.go index 5f90b5eb9..844b1a05e 100644 --- a/extractors/bilibili/types.go +++ b/extractors/bilibili/types.go @@ -143,3 +143,30 @@ type bilibiliWebInterface struct { Code int `json:"code"` Data bilibiliWebInterfaceData `json:"data"` } + +type festival struct { + VideoSections []struct { + Id int64 `json:"id"` + Title string `json:"title"` + Type int `json:"type"` + } `json:"videoSections"` + Episodes []episode `json:"episodes"` + VideoInfo struct { + Aid int `json:"aid"` + BVid string `json:"bvid"` + Cid int `json:"cid"` + Title string `json:"title"` + Desc string `json:"desc"` + Pages []struct { + Cid int `json:"cid"` + Duration int `json:"duration"` + Page int `json:"page"` + Part string `json:"part"` + Dimension struct { + Width int `json:"width"` + Height int `json:"height"` + Rotate int `json:"rotate"` + } `json:"dimension"` + } `json:"pages"` + } `json:"videoInfo"` +} From ee17f58de96b3c7c7486b96e8ffa8c382c3229c6 Mon Sep 17 00:00:00 2001 From: shavit Date: Fri, 9 Feb 2024 11:59:06 -0500 Subject: [PATCH 2/2] Linter recommendations --- downloader/downloader.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index c70e71787..924785b08 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -52,6 +52,10 @@ type Downloader struct { option Options } +const ( + DOWNLOAD_FILE_EXT = ".download" +) + func progressBar(size int64) *pb.ProgressBar { tmpl := `{{counters .}} {{bar . "[" "=" ">" "-" "]"}} {{speed .}} {{percent . | green}} {{rtime .}}` return pb.New64(size). @@ -135,7 +139,7 @@ func (downloader *Downloader) save(part *extractors.Part, refer, fileName string return nil } - tempFilePath := filePath + ".download" + tempFilePath := filePath + DOWNLOAD_FILE_EXT tempFileSize, _, err := utils.FileSize(tempFilePath) if err != nil { return err @@ -233,7 +237,7 @@ func (downloader *Downloader) multiThreadSave(dataPart *extractors.Part, refer, downloader.bar.Add64(fileSize) return nil } - tmpFilePath := filePath + ".download" + tmpFilePath := filePath + DOWNLOAD_FILE_EXT tmpFileSize, tmpExists, err := utils.FileSize(tmpFilePath) if err != nil { return err @@ -469,7 +473,7 @@ func writeFilePartMeta(file *os.File, meta *FilePartMeta) error { } func mergeMultiPart(filepath string, parts []*FilePartMeta) error { - tempFilePath := filepath + ".download" + tempFilePath := filepath + DOWNLOAD_FILE_EXT tempFile, err := os.OpenFile(tempFilePath, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err