diff --git a/PatreonDownloader.App/Models/CommandLineOptions.cs b/PatreonDownloader.App/Models/CommandLineOptions.cs index 4568575..ebe2df7 100644 --- a/PatreonDownloader.App/Models/CommandLineOptions.cs +++ b/PatreonDownloader.App/Models/CommandLineOptions.cs @@ -14,7 +14,8 @@ class CommandLineOptions public bool SaveEmbeds { get; set; } [Option("json", Required = false, HelpText = "Save json data", Default = false)] public bool SaveJson { get; set; } - + [Option("tags", Required = false, HelpText = "Save user defined tags", Default = false)] + public bool SaveTags { get; set; } [Option("campaign-images", Required = false, HelpText = "Download campaign's avatar and cover images", Default = false)] public bool SaveAvatarAndCover { get; set; } diff --git a/PatreonDownloader.App/Program.cs b/PatreonDownloader.App/Program.cs index 760495f..39bd83f 100644 --- a/PatreonDownloader.App/Program.cs +++ b/PatreonDownloader.App/Program.cs @@ -154,6 +154,7 @@ private static async Task InitializeSettings(CommandL SaveDescriptions = commandLineOptions.SaveDescriptions, SaveEmbeds = commandLineOptions.SaveEmbeds, SaveJson = commandLineOptions.SaveJson, + SaveTags = commandLineOptions.SaveTags, DownloadDirectory = commandLineOptions.DownloadDirectory, FileExistsAction = commandLineOptions.FileExistsAction, IsCheckRemoteFileSize = !commandLineOptions.IsDisableRemoteFileSizeCheck, diff --git a/PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs b/PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs index 776e494..340b84f 100644 --- a/PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs +++ b/PatreonDownloader.Implementation/Models/JSONObjects/Posts.cs @@ -112,6 +112,7 @@ public class RootDataAttributes public string Url { get; set; } [JsonProperty("was_posted_by_campaign_owner")] public bool WasPostedByCampaignOwner { get; set; } + } public class AccessRules @@ -185,8 +186,16 @@ public class User public class UserDefinedTags { [JsonProperty("data")] - public List Data { get; set; } + public List Data { get; set; } } + public class UserDefinedTag + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("type")] + public string Type { get; set; } + } public class RootDataRelationships { @@ -218,6 +227,8 @@ public class RootData public RootDataRelationships Relationships { get; set; } [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("user_defined_tags")] + public UserDefinedTags UserDefinedTags { get; set; } } public class ImageUrls diff --git a/PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs b/PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs index 7ae38d8..b370730 100644 --- a/PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs +++ b/PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs @@ -17,6 +17,8 @@ public record PatreonDownloaderSettings : UniversalDownloaderPlatformSettings, I public bool SaveJson { get; init; } + public bool SaveTags { get; init; } + public bool SaveAvatarAndCover { get; init; } /// @@ -60,6 +62,7 @@ public PatreonDownloaderSettings() SaveDescriptions = true; SaveEmbeds = true; SaveJson = true; + SaveTags = true; SaveAvatarAndCover = true; IsUseSubDirectories = false; SubDirectoryPattern = "[%PostId%] %PublishedAt% %PostTitle%"; diff --git a/PatreonDownloader.Implementation/PatreonPageCrawler.cs b/PatreonDownloader.Implementation/PatreonPageCrawler.cs index 45fdd12..c2987f7 100644 --- a/PatreonDownloader.Implementation/PatreonPageCrawler.cs +++ b/PatreonDownloader.Implementation/PatreonPageCrawler.cs @@ -31,7 +31,7 @@ internal sealed class PatreonPageCrawler : IPageCrawler //TODO: Research possibility of not hardcoding this string private const string CrawlStartUrl = "https://www.patreon.com/api/posts?" + - "include=user%2Cattachments%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" + + "include=user%2Cuser_defined_tags%2Cattachments%2Ccampaign%2Cpoll.choices%2Cpoll.current_user_responses.user%2Cpoll.current_user_responses.choice%2Cpoll.current_user_responses.poll%2Caccess_rules.tier.null%2Cimages.null%2Caudio.null" + "&fields[post]=change_visibility_at%2Ccomment_count%2Ccontent%2Ccurrent_user_can_delete%2Ccurrent_user_can_view%2Ccurrent_user_has_liked%2Cembed%2Cimage%2Cis_paid%2Clike_count%2Cmin_cents_pledged_to_view%2Cpost_file%2Cpost_metadata%2Cpublished_at%2Cpatron_count%2Cpatreon_url%2Cpost_type%2Cpledge_url%2Cthumbnail_url%2Cteaser_text%2Ctitle%2Cupgrade_url%2Curl%2Cwas_posted_by_campaign_owner" + "&fields[user]=image_url%2Cfull_name%2Curl" + "&fields[campaign]=show_audio_post_download_links%2Cavatar_photo_url%2Cearnings_visibility%2Cis_nsfw%2Cis_monthly%2Cname%2Curl" + @@ -177,6 +177,32 @@ await File.WriteAllTextAsync(Path.Combine(additionalFilesSaveDirectory, filename } } + if (_patreonDownloaderSettings.SaveTags) +{ + _logger.Error("save tags"); + try + { + string filename = "tags.json"; + _logger.Debug($"[{jsonEntry.Id}] UserDefinedTags: {JsonConvert.SerializeObject(jsonEntry.Relationships.UserDefinedTags)}"); + if(jsonEntry.Relationships.UserDefinedTags != null){ + if (!_patreonDownloaderSettings.IsUseSubDirectories) + filename = $"{jsonEntry.Id}_{filename}"; + + var extractedTags = jsonEntry.Relationships.UserDefinedTags.Data.Select(tag => tag.Id.Split(';').Last()) + .ToList(); + string tagsJson = JsonConvert.SerializeObject(extractedTags, Formatting.Indented); + + await File.WriteAllTextAsync(Path.Combine(additionalFilesSaveDirectory, filename), tagsJson); + } + } + catch (Exception ex) + { + string msg = $"Unable to save tags: {ex}"; + _logger.Error($"[{jsonEntry.Id}] {msg}"); + OnCrawlerMessage(new CrawlerMessageEventArgs(CrawlerMessageType.Error, msg, jsonEntry.Id)); + } +} + if (jsonEntry.Attributes.Embed != null) { if (_patreonDownloaderSettings.SaveEmbeds) @@ -358,7 +384,7 @@ await File.WriteAllTextAsync( jsonEntry.Type != "access-rule" && jsonEntry.Type != "reward" && jsonEntry.Type != "poll_choice" && - jsonEntry.Type != "poll_response") + jsonEntry.Type != "poll_response" && jsonEntry.Type != "post_tag") { string msg = $"Verification for {jsonEntry.Id}: Unknown type for \"included\": {jsonEntry.Type}"; _logger.Error(msg);