From 37a6dbfca453ad90fc47eb91a4420ebd6ce0b56c Mon Sep 17 00:00:00 2001 From: smasterfree Date: Sun, 22 Mar 2020 14:41:16 +0800 Subject: [PATCH] add qingting.fm support --- README.md | 1 + extractors/qingting/qingting.go | 142 +++++++++++++++++++++++++++ extractors/qingting/qingting_test.go | 47 +++++++++ main.go | 3 + utils/utils.go | 2 +- 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 extractors/qingting/qingting.go create mode 100644 extractors/qingting/qingting_test.go diff --git a/README.md b/README.md index c1d137a08..cbd5db6d1 100644 --- a/README.md +++ b/README.md @@ -606,6 +606,7 @@ Pornhub | | ✓ | | | | XVIDEOS | | ✓ | | | | 聯合新聞網 | | ✓ | | | | TikTok | | ✓ | | | | +蜻蜓fm | | | |✓ | | ## Known issues diff --git a/extractors/qingting/qingting.go b/extractors/qingting/qingting.go new file mode 100644 index 000000000..400dfc9dc --- /dev/null +++ b/extractors/qingting/qingting.go @@ -0,0 +1,142 @@ +package qingting + +import ( + "encoding/json" + "fmt" + "github.com/iawia002/annie/config" + "github.com/iawia002/annie/downloader" + "github.com/iawia002/annie/extractors" + "github.com/iawia002/annie/request" + "github.com/iawia002/annie/utils" + "io/ioutil" + "net/http" + "strings" +) + +type ChannelInfoApi struct { + Data ChannelInfo + Code int +} + +type ChannelInfo struct { + ProgramCount int `json:"program_count"` + Name string +} + +type ChannelAudioInfoApi struct { + Data []AudioInfo + Code int + Total int +} + +type AudioInfo struct { + FilePath string `json:"file_path"` + Name string + ResId int `json:"res_id"` + UpdateTime string `json:"update_time"` + Duration int + Playcount string + Id int + Desc string + ChannelId string `json:"channel_id"` + Type string + ImgUrl string `json:"img_url"` +} + +// Extract is the main function for extracting data +func Extract(uri string) ([]downloader.Data, error) { + channelId := extractChannelId(uri) + + // get info of the channel + channelInfoUrl := getChannelInfoUrl(channelId) + channelInfoResponse, err := http.Get(channelInfoUrl) + if err != nil { + fmt.Println("Error in fetching JSON") + return nil, extractors.ErrURLParseFailed + } + defer channelInfoResponse.Body.Close() + channelInfoBody, err := ioutil.ReadAll(channelInfoResponse.Body) + var parsedChannelJson ChannelInfoApi + json.Unmarshal(channelInfoBody, &parsedChannelJson) + + // request API and parse it + audioInfoUrl := getChannelAudioInfoUrl(channelId) + response, err := http.Get(audioInfoUrl) + if err != nil { + fmt.Println("Error in fetching JSON") + return nil, err + } + defer response.Body.Close() + body, err := ioutil.ReadAll(response.Body) + var parsedJson ChannelAudioInfoApi + json.Unmarshal(body, &parsedJson) + + // handle playlist + needDownloadItems := utils.NeedDownloadList(len(parsedJson.Data)) + extractedData := make([]downloader.Data, len(parsedJson.Data)) + wgp := utils.NewWaitGroupPool(config.ThreadNumber) + dataIndex := 0 + for index, audioInfo := range parsedJson.Data { + + if !utils.ItemInSlice(index+1, needDownloadItems) { + continue + } + wgp.Add() + + go func(index int, audioInfo AudioInfo, extractedData []downloader.Data) { + defer wgp.Done() + extractedData[index] = qingtingDownload(audioInfo, uri) + }(dataIndex, audioInfo, extractedData) + dataIndex++ + + } + wgp.Wait() + return extractedData, nil +} + +func extractChannelId(uri string) string { + s := strings.Split(uri, "/") + channid := s[len(s)-1] + return channid +} + +func getChannelInfoUrl(channelId string) string { + return "http://i.qingting.fm/wapi/channels/" + channelId +} + +func getChannelAudioInfoUrl(channelId string) string { + return "http://i.qingting.fm/wapi/channels/" + channelId + "/programs/page/1/pagesize/250" +} + +func getAudioFilePath(filePath string) string { + return "http://od.qingting.fm/" + filePath +} + +func qingtingDownload(audioInfo AudioInfo, uri string) downloader.Data { + title := audioInfo.Name + audioPath := getAudioFilePath(audioInfo.FilePath) + + streams := map[string]downloader.Stream{} + + size, err := request.Size(audioPath, uri) + if err != nil { + return downloader.EmptyData(uri, extractors.ErrURLParseFailed) + } + urlData := downloader.URL{ + URL: audioPath, + Size: size, + Ext: "m4a", + } + streams["default"] = downloader.Stream{ + URLs: []downloader.URL{urlData}, + Size: size, + } + return downloader.Data{ + Site: "qingting fm", + Title: title, + Type: "aduio", + Streams: streams, + URL: uri, + } + +} diff --git a/extractors/qingting/qingting_test.go b/extractors/qingting/qingting_test.go new file mode 100644 index 000000000..1848b9d6b --- /dev/null +++ b/extractors/qingting/qingting_test.go @@ -0,0 +1,47 @@ +package qingting + +import ( + "github.com/iawia002/annie/config" + "github.com/iawia002/annie/downloader" + "github.com/iawia002/annie/test" + "testing" +) + +func TestExtract(t *testing.T) { + config.InfoOnly = true + config.ThreadNumber = 9 + tests := []struct { + name string + args test.Args + playlist bool + }{ + { + name: "playlist test", + args: test.Args{ + URL: "https://www.qingting.fm/channels/226572", + Title: "ViliBili | 这个冬天是个恋爱的季节", + Size: 66284484, + }, + playlist: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ( + data []downloader.Data + err error + ) + if tt.playlist { + // playlist mode + config.Playlist = true + _, err = Extract(tt.args.URL) + test.CheckError(t, err) + } else { + config.Playlist = false + data, err = Extract(tt.args.URL) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + } + }) + } +} diff --git a/main.go b/main.go index 3c4915f51..7e7b229df 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "github.com/iawia002/annie/extractors/netease" "github.com/iawia002/annie/extractors/pixivision" "github.com/iawia002/annie/extractors/pornhub" + "github.com/iawia002/annie/extractors/qingting" "github.com/iawia002/annie/extractors/qq" "github.com/iawia002/annie/extractors/tangdou" "github.com/iawia002/annie/extractors/tiktok" @@ -158,6 +159,8 @@ func download(videoURL string) error { data, err = udn.Extract(videoURL) case "tiktok": data, err = tiktok.Extract(videoURL) + case "qingting": + data, err = qingting.Extract(videoURL) default: data, err = universal.Extract(videoURL) } diff --git a/utils/utils.go b/utils/utils.go index 2052ddf8d..f4340964d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -73,7 +73,7 @@ func Domain(url string) string { domainPattern := `([a-z0-9][-a-z0-9]{0,62})\.` + `(com\.cn|com\.hk|` + `cn|com|net|edu|gov|biz|org|info|pro|name|xxx|xyz|be|` + - `me|top|cc|tv|tt)` + `me|top|cc|tv|tt|fm)` domain := MatchOneOf(url, domainPattern) if domain != nil { return domain[1]