From e33fa926dd846efb1394a2e5c39a35f888ce3db3 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 4 Dec 2022 13:19:24 +0100 Subject: [PATCH 1/6] [SoundCloud] Add support for comment replies --- .../org/schabi/newpipe/extractor/Page.java | 36 +++++++- .../soundcloud/SoundcloudParsingHelper.java | 13 +++ .../SoundcloudCommentsExtractor.java | 73 ++++++++++++---- .../SoundcloudCommentsInfoItemExtractor.java | 85 ++++++++++++++++--- .../SoundcloudCommentsLinkHandlerFactory.java | 22 ++++- .../SoundcloudStreamLinkHandlerFactory.java | 2 +- 6 files changed, 201 insertions(+), 30 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java index e1b19e7fb9..e13a922878 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java @@ -1,18 +1,26 @@ package org.schabi.newpipe.extractor; +import javax.annotation.Nullable; import java.io.Serializable; import java.util.List; import java.util.Map; -import javax.annotation.Nullable; - import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +/** + * The {@link Page} class is used for storing information on future requests + * for retrieving content. + *
+ * A page has an {@link #id}, an {@link #url}, as well as information on possible {@link #cookies}. + * In case the data behind the URL has already been retrieved, + * it can be accessed by using @link #getBody()} and {@link #getContent()}. + */ public class Page implements Serializable { private final String url; private final String id; private final List ids; private final Map cookies; + private Serializable content; @Nullable private final byte[] body; @@ -78,4 +86,28 @@ public static boolean isValid(final Page page) { public byte[] getBody() { return body; } + + public boolean hasContent() { + return content != null; + } + + /** + * Get the page's content if it has been set, returns {@code null} otherwise. + * @return the page's content + */ + @Nullable + public Serializable getContent() { + return content; + } + + /** + * Set the page's content. + * The page's content can either be retrieved manually by requesting the resource + * behind the page's URL (see {@link #url} and {@link #getUrl()}) + * or storing it in a {@link Page}s instance in case the content has already been downloaded. + * @param content the page's content + */ + public void setContent(@Nullable final Serializable content) { + this.content = content; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 57deb64a21..3d217ce80c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -323,4 +323,17 @@ public static String getAvatarUrl(final JsonObject object) { public static String getUploaderName(final JsonObject object) { return object.getObject("user").getString("username", ""); } + + public static boolean isReplyTo(@Nonnull final JsonObject originalComment, + @Nonnull final JsonObject otherComment) { + final String mention = "@" + originalComment.getObject("user").getString("permalink"); + return otherComment.getString("body").startsWith(mention) + && originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); + + } + + public static boolean isReply(@Nonnull final JsonObject comment) { + return comment.getString("body").startsWith("@"); + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index b02a3ea802..d4afe9f09c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -16,6 +16,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; +import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import java.io.IOException; @@ -24,6 +25,8 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; public class SoundcloudCommentsExtractor extends CommentsExtractor { + public static final String COLLECTION = "collection"; + public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); @@ -46,7 +49,7 @@ public InfoItemsPage getInitialPage() throws ExtractionExcepti final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json.getArray("collection")); + collectStreamsFrom(collector, json); return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); } @@ -57,21 +60,32 @@ public InfoItemsPage getPage(final Page page) throws Extractio if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } - - final Downloader downloader = NewPipe.getDownloader(); - final Response response = downloader.get(page.getUrl()); - final JsonObject json; - try { - json = JsonParser.object().from(response.responseBody()); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse json", e); - } - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json.getArray("collection")); + if (page.hasContent()) { + // This page contains the whole previously fetched comments. + // We need to get the comments which are replies to the comment with the page's id. + json = (JsonObject) page.getContent(); + try { + final int commentId = Integer.parseInt(page.getId()); + collectRepliesFrom(collector, json, commentId, page.getUrl()); + } catch (final NumberFormatException e) { + throw new ParsingException("Got invalid comment id", e); + } + } else { + + final Downloader downloader = NewPipe.getDownloader(); + final Response response = downloader.get(page.getUrl()); + + try { + json = JsonParser.object().from(response.responseBody()); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse json", e); + } + collectStreamsFrom(collector, json); + } return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); } @@ -80,10 +94,39 @@ public InfoItemsPage getPage(final Page page) throws Extractio public void onFetchPage(@Nonnull final Downloader downloader) { } private void collectStreamsFrom(final CommentsInfoItemsCollector collector, - final JsonArray entries) throws ParsingException { + final JsonObject json) throws ParsingException { final String url = getUrl(); - for (final Object comment : entries) { - collector.commit(new SoundcloudCommentsInfoItemExtractor((JsonObject) comment, url)); + final JsonArray entries = json.getArray(COLLECTION); + for (int i = 0; i < entries.size(); i++) { + final JsonObject entry = entries.getObject(i); + if (i == 0 + || (!SoundcloudParsingHelper.isReply(entry) + && !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry))) { + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, entries.getObject(i), url)); + } } } + + private void collectRepliesFrom(final CommentsInfoItemsCollector collector, + final JsonObject json, + final int id, + final String url) throws ParsingException { + JsonObject originalComment = null; + final JsonArray entries = json.getArray(COLLECTION); + for (int i = 0; i < entries.size(); i++) { + final JsonObject comment = entries.getObject(i); + if (comment.getInt("id") == id) { + originalComment = comment; + continue; + } + if (originalComment != null + && SoundcloudParsingHelper.isReplyTo(originalComment, comment)) { + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, entries.getObject(i), url)); + + } + } + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index ec3f353e62..79b27f1b2d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,62 +1,79 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; +import org.schabi.newpipe.extractor.Page; +import org.schabi.newpipe.extractor.ServiceList; +import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; +import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + public static final String USER = "user"; + public static final String BODY = "body"; + private final JsonObject json; + private final int index; + private final JsonObject item; private final String url; - public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final String url) { + private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; + private Page repliesPage = null; + + public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, final JsonObject item, final String url) { this.json = json; + this.index = index; + this.item = item; this.url = url; } @Override public String getCommentId() { - return Objects.toString(json.getLong("id"), null); + return Objects.toString(item.getLong("id"), null); } @Override public Description getCommentText() { - return new Description(json.getString("body"), Description.PLAIN_TEXT); + return new Description(item.getString(BODY), Description.PLAIN_TEXT); } @Override public String getUploaderName() { - return json.getObject("user").getString("username"); + return item.getObject(USER).getString("username"); } @Override public String getUploaderAvatarUrl() { - return json.getObject("user").getString("avatar_url"); + return item.getObject(USER).getString("avatar_url"); } @Override public boolean isUploaderVerified() throws ParsingException { - return json.getObject("user").getBoolean("verified"); + return item.getObject(USER).getBoolean("verified"); } @Override public int getStreamPosition() throws ParsingException { - return json.getInt("timestamp") / 1000; // convert milliseconds to seconds + return item.getInt("timestamp") / 1000; // convert milliseconds to seconds } @Override public String getUploaderUrl() { - return json.getObject("user").getString("permalink_url"); + return item.getObject(USER).getString("permalink_url"); } @Override public String getTextualUploadDate() { - return json.getString("created_at"); + return item.getString("created_at"); } @Nullable @@ -67,7 +84,7 @@ public DateWrapper getUploadDate() throws ParsingException { @Override public String getName() throws ParsingException { - return json.getObject("user").getString("permalink"); + return item.getObject(USER).getString("permalink"); } @Override @@ -77,6 +94,52 @@ public String getUrl() { @Override public String getThumbnailUrl() { - return json.getObject("user").getString("avatar_url"); + return item.getObject(USER).getString("avatar_url"); + } + + @Override + public Page getReplies() { + if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { + final List replies = new ArrayList<>(); + final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( + ServiceList.SoundCloud.getServiceId()); + final JsonArray jsonArray = new JsonArray(); + // Replies start with the mention of the user who created the original comment. + final String mention = "@" + item.getObject(USER).getString("permalink"); + // Loop through all comments which come after the original comment to find its replies. + final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + for (int i = index + 1; i < allItems.size(); i++) { + final JsonObject comment = allItems.getObject(i); + final String commentContent = comment.getString("body"); + if (commentContent.startsWith(mention)) { + replies.add(comment); + jsonArray.add(comment); + collector.commit(new SoundcloudCommentsInfoItemExtractor(json, i, comment, url)); + } else if (!commentContent.startsWith("@") || replies.isEmpty()) { + // Only the comments directly after the original comment + // starting with the mention of the comment's creator + // are replies to the original comment. + // The first comment not starting with these letters + // is the next top-level comment. + break; + } + } + replyCount = jsonArray.size(); + if (collector.getItems().isEmpty()) { + return null; + } + repliesPage = new Page(getUrl(), getCommentId()); + repliesPage.setContent(json); + } + + return repliesPage; + } + + @Override + public int getReplyCount() throws ParsingException { + if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { + getReplies(); + } + return replyCount; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java index 23c6a29392..39e124698a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -3,6 +3,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandlerFactory; +import org.schabi.newpipe.extractor.utils.Parser; import java.io.IOException; import java.util.List; @@ -14,6 +15,8 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF private static final SoundcloudCommentsLinkHandlerFactory INSTANCE = new SoundcloudCommentsLinkHandlerFactory(); + private static final String OFFSET_PATTERN = "https://api-v2.soundcloud.com/tracks/([0-9a-z]+)/comments?([0-9a-z/&])?offset=([0-9])+" + private SoundcloudCommentsLinkHandlerFactory() { } @@ -27,7 +30,7 @@ public String getUrl(final String id, final String sortFilter) throws ParsingException { try { return "https://api-v2.soundcloud.com/tracks/" + id + "/comments" + "?client_id=" - + clientId() + "&threaded=0" + "&filter_replies=1"; + + clientId() + "&threaded=1" + "&filter_replies=1"; // Anything but 1 = sort by new // + "&limit=NUMBER_OF_ITEMS_PER_REQUEST". We let the API control (default = 10) // + "&offset=OFFSET". We let the API control (default = 0, then we use nextPageUrl) @@ -36,12 +39,29 @@ public String getUrl(final String id, } } + public String getUrl(final String id, + final List contentFilter, + final String sortFilter, + final int offset) throws ParsingException { + return getUrl(id, contentFilter, sortFilter) + "&offset=" + offset; + } + @Override public String getId(final String url) throws ParsingException { // Delegation to avoid duplicate code, as we need the same id return SoundcloudStreamLinkHandlerFactory.getInstance().getId(url); } + public int getReplyOffset(final String url) throws ParsingException { + try { + return Integer.parseInt(Parser.matchGroup(OFFSET_PATTERN, url, 3)); + } catch (Parser.RegexException | NumberFormatException e) { + throw new ParsingException("Could not get offset from URL: " + url, e); + } + } + + + @Override public boolean onAcceptUrl(final String url) { try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 9af4be09b9..14ee29b0c3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -33,7 +33,7 @@ public String getUrl(final String id) throws ParsingException { @Override public String getId(final String url) throws ParsingException { if (Parser.isMatch(API_URL_PATTERN, url)) { - return Parser.matchGroup1(API_URL_PATTERN, url); + return Parser.matchGroup(API_URL_PATTERN, url, 2); } Utils.checkUrl(URL_PATTERN, url); From d41a4a1350ab8ffa4f4f26147d47dc8ed0b7669d Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sat, 3 Dec 2022 16:29:52 +0100 Subject: [PATCH 2/6] Only display direct replies as replies --- .../SoundcloudCommentsExtractor.java | 23 +++++++++++++++---- .../SoundcloudCommentsLinkHandlerFactory.java | 3 ++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index d4afe9f09c..d51a5fbc94 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -64,13 +64,15 @@ public InfoItemsPage getPage(final Page page) throws Extractio final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); + // Replies typically do not have a next page, but that's not always the case. + final boolean hasNextPage; if (page.hasContent()) { // This page contains the whole previously fetched comments. // We need to get the comments which are replies to the comment with the page's id. json = (JsonObject) page.getContent(); try { final int commentId = Integer.parseInt(page.getId()); - collectRepliesFrom(collector, json, commentId, page.getUrl()); + hasNextPage = collectRepliesFrom(collector, json, commentId, page.getUrl()); } catch (final NumberFormatException e) { throw new ParsingException("Got invalid comment id", e); } @@ -81,13 +83,18 @@ public InfoItemsPage getPage(final Page page) throws Extractio try { json = JsonParser.object().from(response.responseBody()); + hasNextPage = json.has("next_href"); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } collectStreamsFrom(collector, json); } - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + if (hasNextPage) { + return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + } else { + return new InfoItemsPage<>(collector, null); + } } @Override @@ -108,12 +115,13 @@ private void collectStreamsFrom(final CommentsInfoItemsCollector collector, } } - private void collectRepliesFrom(final CommentsInfoItemsCollector collector, + private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, final JsonObject json, final int id, final String url) throws ParsingException { JsonObject originalComment = null; final JsonArray entries = json.getArray(COLLECTION); + boolean moreReplies = false; for (int i = 0; i < entries.size(); i++) { final JsonObject comment = entries.getObject(i); if (comment.getInt("id") == id) { @@ -123,10 +131,15 @@ private void collectRepliesFrom(final CommentsInfoItemsCollector collector, if (originalComment != null && SoundcloudParsingHelper.isReplyTo(originalComment, comment)) { collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, entries.getObject(i), url)); - + json, i, entries.getObject(i), url, originalComment)); + // There might be more replies to the originalComment, + // especially if the original comment is at the end of the list. + if (i == entries.size() - 1 && json.has("next_href")) { + moreReplies = true; + } } } + return moreReplies; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java index 39e124698a..775ee10486 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudCommentsLinkHandlerFactory.java @@ -15,7 +15,8 @@ public final class SoundcloudCommentsLinkHandlerFactory extends ListLinkHandlerF private static final SoundcloudCommentsLinkHandlerFactory INSTANCE = new SoundcloudCommentsLinkHandlerFactory(); - private static final String OFFSET_PATTERN = "https://api-v2.soundcloud.com/tracks/([0-9a-z]+)/comments?([0-9a-z/&])?offset=([0-9])+" + private static final String OFFSET_PATTERN = "https://api-v2.soundcloud.com/tracks/" + + "([0-9a-z]+)/comments?([0-9a-z/&])?offset=([0-9])+"; private SoundcloudCommentsLinkHandlerFactory() { } From e9bbc5dace095777c6ca0361506f2662647a95b0 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 4 Dec 2022 14:32:50 +0100 Subject: [PATCH 3/6] Use full names for comment authors and their mentions in replies if available --- .../SoundcloudCommentsInfoItemExtractor.java | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index 79b27f1b2d..a6d2b9a7fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -20,20 +20,30 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public static final String USER = "user"; public static final String BODY = "body"; + public static final String USER_PERMALINK = "permalink"; private final JsonObject json; private final int index; private final JsonObject item; private final String url; + private final JsonObject superComment; private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; private Page repliesPage = null; - public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, final JsonObject item, final String url) { + public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, + final JsonObject item, final String url, + @Nullable final JsonObject superComment) { this.json = json; this.index = index; this.item = item; this.url = url; + this.superComment = superComment; + } + + public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, + final JsonObject item, final String url) { + this(json, index, item, url, null); } @Override @@ -43,12 +53,30 @@ public String getCommentId() { @Override public Description getCommentText() { - return new Description(item.getString(BODY), Description.PLAIN_TEXT); + String commentContent = item.getString(BODY); + if (superComment == null) { + return new Description(commentContent, Description.PLAIN_TEXT); + } + // This comment is a reply to another comment. + // Therefore, the comment starts with the mention of the original comment's author. + // The account is automatically linked by the SoundCloud web UI. + // We need to do this manually. + final JsonObject user = superComment.getObject("user"); + final String link = "" + + "@" + user.getString("full_name") + ""; + commentContent = commentContent + .replace("@" + user.getString(USER_PERMALINK), link) + .replace("@" + superComment.getInt("user_id"), link); + + return new Description(commentContent, Description.HTML); } @Override public String getUploaderName() { - return item.getObject(USER).getString("username"); + if (isNullOrEmpty(user.getString("full_name"))) { + return user.getString("username"); + } + return user.getString("full_name"); } @Override @@ -105,7 +133,7 @@ public Page getReplies() { ServiceList.SoundCloud.getServiceId()); final JsonArray jsonArray = new JsonArray(); // Replies start with the mention of the user who created the original comment. - final String mention = "@" + item.getObject(USER).getString("permalink"); + final String mention = "@" + item.getObject(USER).getString(USER_PERMALINK); // Loop through all comments which come after the original comment to find its replies. final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); for (int i = index + 1; i < allItems.size(); i++) { @@ -114,7 +142,8 @@ public Page getReplies() { if (commentContent.startsWith(mention)) { replies.add(comment); jsonArray.add(comment); - collector.commit(new SoundcloudCommentsInfoItemExtractor(json, i, comment, url)); + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, comment, url, item)); } else if (!commentContent.startsWith("@") || replies.isEmpty()) { // Only the comments directly after the original comment // starting with the mention of the comment's creator From b6e3015ee2f40752a4033fa676031e11c6309f65 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Sun, 4 Dec 2022 16:54:45 +0100 Subject: [PATCH 4/6] Fix displaying replies which do not start with the mention of another user --- .../soundcloud/SoundcloudParsingHelper.java | 8 +- .../SoundcloudCommentsExtractor.java | 31 ++--- .../SoundcloudCommentsInfoItemExtractor.java | 106 +++++++++++------- 3 files changed, 82 insertions(+), 63 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 3d217ce80c..8a1abc68c6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -326,14 +326,8 @@ public static String getUploaderName(final JsonObject object) { public static boolean isReplyTo(@Nonnull final JsonObject originalComment, @Nonnull final JsonObject otherComment) { - final String mention = "@" + originalComment.getObject("user").getString("permalink"); - return otherComment.getString("body").startsWith(mention) - && originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); + return originalComment.getInt("timestamp") == otherComment.getInt("timestamp"); } - public static boolean isReply(@Nonnull final JsonObject comment) { - return comment.getString("body").startsWith("@"); - } - } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index d51a5fbc94..f253cb695b 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -1,5 +1,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonParser; @@ -22,10 +24,9 @@ import javax.annotation.Nonnull; -import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; - public class SoundcloudCommentsExtractor extends CommentsExtractor { public static final String COLLECTION = "collection"; + public static final String NEXT_HREF = "next_href"; public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { @@ -49,9 +50,9 @@ public InfoItemsPage getInitialPage() throws ExtractionExcepti final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectStreamsFrom(collector, json); + collectCommentsFrom(collector, json); - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } @Override @@ -83,15 +84,15 @@ public InfoItemsPage getPage(final Page page) throws Extractio try { json = JsonParser.object().from(response.responseBody()); - hasNextPage = json.has("next_href"); + hasNextPage = json.has(NEXT_HREF); } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - collectStreamsFrom(collector, json); + collectCommentsFrom(collector, json); } if (hasNextPage) { - return new InfoItemsPage<>(collector, new Page(json.getString("next_href"))); + return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } else { return new InfoItemsPage<>(collector, null); } @@ -100,17 +101,19 @@ public InfoItemsPage getPage(final Page page) throws Extractio @Override public void onFetchPage(@Nonnull final Downloader downloader) { } - private void collectStreamsFrom(final CommentsInfoItemsCollector collector, - final JsonObject json) throws ParsingException { + private void collectCommentsFrom(final CommentsInfoItemsCollector collector, + final JsonObject json) throws ParsingException { final String url = getUrl(); final JsonArray entries = json.getArray(COLLECTION); + JsonObject lastTopComment = null; for (int i = 0; i < entries.size(); i++) { final JsonObject entry = entries.getObject(i); if (i == 0 - || (!SoundcloudParsingHelper.isReply(entry) - && !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry))) { + || (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) + && !SoundcloudParsingHelper.isReplyTo(lastTopComment, entry))) { + lastTopComment = entry; collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, entries.getObject(i), url)); + json, i, entry, url)); } } } @@ -118,7 +121,7 @@ private void collectStreamsFrom(final CommentsInfoItemsCollector collector, private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, final JsonObject json, final int id, - final String url) throws ParsingException { + final String url) { JsonObject originalComment = null; final JsonArray entries = json.getArray(COLLECTION); boolean moreReplies = false; @@ -134,7 +137,7 @@ private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, json, i, entries.getObject(i), url, originalComment)); // There might be more replies to the originalComment, // especially if the original comment is at the end of the list. - if (i == entries.size() - 1 && json.has("next_href")) { + if (i == entries.size() - 1 && json.has(NEXT_HREF)) { moreReplies = true; } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index a6d2b9a7fd..db9ef549dd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,7 +1,10 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; + import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; + import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; @@ -12,20 +15,21 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import org.schabi.newpipe.extractor.stream.Description; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; +import javax.annotation.Nullable; + public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { - public static final String USER = "user"; public static final String BODY = "body"; public static final String USER_PERMALINK = "permalink"; + public static final String USER_FULL_NAME = "full_name"; + public static final String USER_USERNAME = "username"; private final JsonObject json; private final int index; private final JsonObject item; private final String url; + private final JsonObject user; private final JsonObject superComment; private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; @@ -39,6 +43,7 @@ public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int inde this.item = item; this.url = url; this.superComment = superComment; + this.user = item.getObject("user"); } public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, @@ -50,7 +55,6 @@ public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int inde public String getCommentId() { return Objects.toString(item.getLong("id"), null); } - @Override public Description getCommentText() { String commentContent = item.getString(BODY); @@ -61,32 +65,49 @@ public Description getCommentText() { // Therefore, the comment starts with the mention of the original comment's author. // The account is automatically linked by the SoundCloud web UI. // We need to do this manually. - final JsonObject user = superComment.getObject("user"); - final String link = "" - + "@" + user.getString("full_name") + ""; - commentContent = commentContent - .replace("@" + user.getString(USER_PERMALINK), link) - .replace("@" + superComment.getInt("user_id"), link); + if (commentContent.startsWith("@")) { + final String authorName = commentContent.split(" ", 2)[0].replace("@", ""); + final JsonArray comments = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + JsonObject author = null; + for (int i = index - 1; i >= 0 && author == null; i--) { + final JsonObject commentsAuthor = comments.getObject(i).getObject("user"); + // use startsWith because sometimes the mention of the user + // is followed by a punctuation character. + if (authorName.startsWith(commentsAuthor.getString(USER_PERMALINK))) { + author = commentsAuthor; + } + } + if (author == null) { + author = superComment.getObject("user"); + } + final String name = isNullOrEmpty(author.getString(USER_FULL_NAME)) + ? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME); + final String link = "" + + "@" + name + ""; + commentContent = commentContent + .replace("@" + author.getString(USER_PERMALINK), link) + .replace("@" + author.getInt("user_id"), link); + } return new Description(commentContent, Description.HTML); } @Override public String getUploaderName() { - if (isNullOrEmpty(user.getString("full_name"))) { - return user.getString("username"); + if (isNullOrEmpty(user.getString(USER_FULL_NAME))) { + return user.getString(USER_USERNAME); } - return user.getString("full_name"); + return user.getString(USER_FULL_NAME); } @Override public String getUploaderAvatarUrl() { - return item.getObject(USER).getString("avatar_url"); + return user.getString("avatar_url"); } @Override public boolean isUploaderVerified() throws ParsingException { - return item.getObject(USER).getBoolean("verified"); + return user.getBoolean("verified"); } @Override @@ -96,7 +117,7 @@ public int getStreamPosition() throws ParsingException { @Override public String getUploaderUrl() { - return item.getObject(USER).getString("permalink_url"); + return user.getString("permalink_url"); } @Override @@ -112,7 +133,7 @@ public DateWrapper getUploadDate() throws ParsingException { @Override public String getName() throws ParsingException { - return item.getObject(USER).getString("permalink"); + return user.getString(USER_PERMALINK); } @Override @@ -122,38 +143,39 @@ public String getUrl() { @Override public String getThumbnailUrl() { - return item.getObject(USER).getString("avatar_url"); + return user.getString("avatar_url"); } @Override public Page getReplies() { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { - final List replies = new ArrayList<>(); + final JsonArray replies = new JsonArray(); final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( ServiceList.SoundCloud.getServiceId()); - final JsonArray jsonArray = new JsonArray(); - // Replies start with the mention of the user who created the original comment. - final String mention = "@" + item.getObject(USER).getString(USER_PERMALINK); - // Loop through all comments which come after the original comment to find its replies. - final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); - for (int i = index + 1; i < allItems.size(); i++) { - final JsonObject comment = allItems.getObject(i); - final String commentContent = comment.getString("body"); - if (commentContent.startsWith(mention)) { - replies.add(comment); - jsonArray.add(comment); - collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, comment, url, item)); - } else if (!commentContent.startsWith("@") || replies.isEmpty()) { - // Only the comments directly after the original comment - // starting with the mention of the comment's creator - // are replies to the original comment. - // The first comment not starting with these letters - // is the next top-level comment. - break; + // SoundCloud has only comments and top level replies, but not nested replies. + // Therefore, replies cannot have further replies. + if (superComment == null) { + // Loop through all comments which come after the original comment + // to find its replies. + final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + boolean foundReply = false; + for (int i = index + 1; i < allItems.size(); i++) { + final JsonObject comment = allItems.getObject(i); + if (SoundcloudParsingHelper.isReplyTo(item, comment)) { + replies.add(comment); + collector.commit(new SoundcloudCommentsInfoItemExtractor( + json, i, comment, url, item)); + foundReply = true; + } else if (foundReply) { + // Only the comments directly after the original comment + // having the same timestamp are replies to the original comment. + // The first comment not having the same timestamp + // is the next top-level comment. + break; + } } } - replyCount = jsonArray.size(); + replyCount = replies.size(); if (collector.getItems().isEmpty()) { return null; } @@ -165,7 +187,7 @@ public Page getReplies() { } @Override - public int getReplyCount() throws ParsingException { + public int getReplyCount() { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { getReplies(); } From e5be686b06ddeda6a5d0fd3d18521930ae6a5674 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 2 Jan 2023 18:59:03 +0100 Subject: [PATCH 5/6] Try to fix detecting replies to a comment on the previous page When getting a page which is not the initial page there it is possible that the first comments are replies to a comment from a previous page. --- .../org/schabi/newpipe/extractor/Page.java | 2 +- .../SoundcloudCommentsExtractor.java | 103 +++++++++++++++--- .../SoundcloudCommentsInfoItemExtractor.java | 54 ++++----- 3 files changed, 115 insertions(+), 44 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java index e13a922878..091c5e7675 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Page.java @@ -13,7 +13,7 @@ *
* A page has an {@link #id}, an {@link #url}, as well as information on possible {@link #cookies}. * In case the data behind the URL has already been retrieved, - * it can be accessed by using @link #getBody()} and {@link #getContent()}. + * it can be accessed by using {@link #getBody()} or {@link #getContent()}. */ public class Page implements Serializable { private final String url; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index f253cb695b..56af6b43eb 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -21,13 +21,24 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class SoundcloudCommentsExtractor extends CommentsExtractor { public static final String COLLECTION = "collection"; public static final String NEXT_HREF = "next_href"; + /** + * The last comment which was a top level comment. + * Next pages might start with replies to the last top level comment + * and therefore the {@link SoundcloudCommentsInfoItemExtractor#replyCount} + * of the last top level comment cannot be determined certainly. + */ + @Nullable private JsonObject lastTopLevelComment; + public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { super(service, uiHandler); @@ -50,14 +61,15 @@ public InfoItemsPage getInitialPage() throws ExtractionExcepti final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( getServiceId()); - collectCommentsFrom(collector, json); + collectCommentsFrom(collector, json, null); return new InfoItemsPage<>(collector, new Page(json.getString(NEXT_HREF))); } @Override - public InfoItemsPage getPage(final Page page) throws ExtractionException, - IOException { + public InfoItemsPage getPage(final Page page) + throws ExtractionException, IOException { + if (page == null || isNullOrEmpty(page.getUrl())) { throw new IllegalArgumentException("Page doesn't contain an URL"); } @@ -88,7 +100,7 @@ public InfoItemsPage getPage(final Page page) throws Extractio } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - collectCommentsFrom(collector, json); + collectCommentsFrom(collector, json, lastTopLevelComment); } if (hasNextPage) { @@ -101,27 +113,86 @@ public InfoItemsPage getPage(final Page page) throws Extractio @Override public void onFetchPage(@Nonnull final Downloader downloader) { } - private void collectCommentsFrom(final CommentsInfoItemsCollector collector, - final JsonObject json) throws ParsingException { + /** + * Collect top level comments from a SoundCloud API response. + * @param collector the collector which collects the the top level comments + * @param json the JsonObject of the API response + * @param lastTopLevelComment the last top level comment from the previous page or {@code null} + * if this method is run for the initial page. + * @throws ParsingException + */ + private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector collector, + @Nonnull final JsonObject json, + @Nullable final JsonObject lastTopLevelComment) + throws ParsingException { + final List extractors = new ArrayList<>(); final String url = getUrl(); final JsonArray entries = json.getArray(COLLECTION); - JsonObject lastTopComment = null; + /** + * The current top level comment. + */ + JsonObject currentTopLevelComment = null; + boolean isLastCommentReply = true; + // Check whether the first comment in the list is a reply to the last top level comment + // from the previous page if there was a previous page. + if (lastTopLevelComment != null) { + final JsonObject firstComment = entries.getObject(0); + if (SoundcloudParsingHelper.isReplyTo(lastTopLevelComment, firstComment)) { + currentTopLevelComment = lastTopLevelComment; + } else { + extractors.add(new SoundcloudCommentsInfoItemExtractor( + json, SoundcloudCommentsInfoItemExtractor.PREVIOUS_PAGE_INDEX, + firstComment, url, null)); + } + } + for (int i = 0; i < entries.size(); i++) { final JsonObject entry = entries.getObject(i); - if (i == 0 + // extract all top level comments + // The first comment is either a top level comment + // if it is not a reply to the last top level comment + // + if (i == 0 && currentTopLevelComment == null || (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) - && !SoundcloudParsingHelper.isReplyTo(lastTopComment, entry))) { - lastTopComment = entry; - collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, entry, url)); + && !SoundcloudParsingHelper.isReplyTo(currentTopLevelComment, entry))) { + currentTopLevelComment = entry; + if (i == entries.size() - 1) { + isLastCommentReply = false; + this.lastTopLevelComment = currentTopLevelComment; + // Do not collect the last comment if it is a top level comment + // because it might have replies. + // That is information we cannot get from the comment itself + // (thanks SoundCloud...) but needs to be obtained from the next comment. + // The comment will therefore be collected + // when collecting the items from the next page. + break; + } + extractors.add(new SoundcloudCommentsInfoItemExtractor( + json, i, entry, url, lastTopLevelComment)); } } + if (isLastCommentReply) { + // Do not collect the last top level comment if it has replies and the retrieved + // comment list ends with a reply. We do not know whether the next page starts + // with more replies to the last top level comment. + this.lastTopLevelComment = extractors.remove(extractors.size() - 1).item; + } + extractors.stream().forEach(collector::commit); + } - private boolean collectRepliesFrom(final CommentsInfoItemsCollector collector, - final JsonObject json, - final int id, - final String url) { + /** + * Collect replies to a top level comment from a SoundCloud API response. + * @param collector the collector which collects the the replies + * @param json the SoundCloud API response + * @param id the comment's id for which the replies are collected + * @param url the corresponding page's URL + * @return + */ + private boolean collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector collector, + @Nonnull final JsonObject json, + final int id, + @Nonnull final String url) { JsonObject originalComment = null; final JsonArray entries = json.getArray(COLLECTION); boolean moreReplies = false; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index db9ef549dd..0dfa3edeee 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -6,10 +6,8 @@ import com.grack.nanojson.JsonObject; import org.schabi.newpipe.extractor.Page; -import org.schabi.newpipe.extractor.ServiceList; import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; -import org.schabi.newpipe.extractor.comments.CommentsInfoItemsCollector; import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; @@ -17,32 +15,42 @@ import java.util.Objects; +import javax.annotation.Nonnull; import javax.annotation.Nullable; public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor { + public static final int PREVIOUS_PAGE_INDEX = -1; public static final String BODY = "body"; public static final String USER_PERMALINK = "permalink"; public static final String USER_FULL_NAME = "full_name"; public static final String USER_USERNAME = "username"; - private final JsonObject json; + @Nonnull private final JsonObject json; private final int index; - private final JsonObject item; + @Nonnull public final JsonObject item; private final String url; - private final JsonObject user; - private final JsonObject superComment; - + @Nonnull private final JsonObject user; + /** + * A comment to which this comment is a reply. + * Is {@code null} if this comment is itself a top level comment. + */ + @Nullable private final JsonObject topLevelComment; + + /** + * The reply count is not given by the SoundCloud API, but needs to be obtained + * by counting the comments which come directly after this item and have the same timestamp. + */ private int replyCount = CommentsInfoItem.UNKNOWN_REPLY_COUNT; private Page repliesPage = null; - public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int index, - final JsonObject item, final String url, - @Nullable final JsonObject superComment) { + public SoundcloudCommentsInfoItemExtractor(@Nonnull final JsonObject json, final int index, + @Nonnull final JsonObject item, final String url, + @Nullable final JsonObject topLevelComment) { this.json = json; this.index = index; this.item = item; this.url = url; - this.superComment = superComment; + this.topLevelComment = topLevelComment; this.user = item.getObject("user"); } @@ -58,7 +66,7 @@ public String getCommentId() { @Override public Description getCommentText() { String commentContent = item.getString(BODY); - if (superComment == null) { + if (topLevelComment == null) { return new Description(commentContent, Description.PLAIN_TEXT); } // This comment is a reply to another comment. @@ -78,7 +86,7 @@ public Description getCommentText() { } } if (author == null) { - author = superComment.getObject("user"); + author = topLevelComment.getObject("user"); } final String name = isNullOrEmpty(author.getString(USER_FULL_NAME)) ? author.getString(USER_USERNAME) : author.getString(USER_FULL_NAME); @@ -149,24 +157,17 @@ public String getThumbnailUrl() { @Override public Page getReplies() { if (replyCount == CommentsInfoItem.UNKNOWN_REPLY_COUNT) { - final JsonArray replies = new JsonArray(); - final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector( - ServiceList.SoundCloud.getServiceId()); + replyCount = 0; // SoundCloud has only comments and top level replies, but not nested replies. // Therefore, replies cannot have further replies. - if (superComment == null) { + if (topLevelComment == null) { // Loop through all comments which come after the original comment // to find its replies. final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); - boolean foundReply = false; for (int i = index + 1; i < allItems.size(); i++) { - final JsonObject comment = allItems.getObject(i); - if (SoundcloudParsingHelper.isReplyTo(item, comment)) { - replies.add(comment); - collector.commit(new SoundcloudCommentsInfoItemExtractor( - json, i, comment, url, item)); - foundReply = true; - } else if (foundReply) { + if (SoundcloudParsingHelper.isReplyTo(item, allItems.getObject(i))) { + replyCount++; + } else { // Only the comments directly after the original comment // having the same timestamp are replies to the original comment. // The first comment not having the same timestamp @@ -175,8 +176,7 @@ public Page getReplies() { } } } - replyCount = replies.size(); - if (collector.getItems().isEmpty()) { + if (replyCount == 0) { return null; } repliesPage = new Page(getUrl(), getCommentId()); From 8ae7fcfa1e9276478f1170796ff751894a22fe39 Mon Sep 17 00:00:00 2001 From: TobiGr Date: Tue, 3 Jan 2023 00:19:41 +0100 Subject: [PATCH 6/6] Almost fixed Implemented a cache. TODO: Do not store in cache when viewing replies.... --- .../newpipe/extractor/InfoItemsCollector.java | 2 +- .../SoundcloudCommentsExtractor.java | 108 +++++++++++++----- .../SoundcloudCommentsInfoItemExtractor.java | 12 +- .../YoutubeDashManifestCreatorsUtils.java | 2 +- .../YoutubeOtfDashManifestCreator.java | 2 +- ...ePostLiveStreamDvrDashManifestCreator.java | 2 +- ...YoutubeProgressiveDashManifestCreator.java | 2 +- .../newpipe/extractor/utils/cache/Cache.java | 9 ++ .../{ => cache}/ManifestCreatorCache.java | 4 +- .../utils/cache/SoundCloudCommentsCache.java | 74 ++++++++++++ .../{ => cache}/ManifestCreatorCacheTest.java | 3 +- .../cache/SoundCloudCommentsCacheTest.java | 83 ++++++++++++++ 12 files changed, 264 insertions(+), 39 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java rename extractor/src/main/java/org/schabi/newpipe/extractor/utils/{ => cache}/ManifestCreatorCache.java (98%) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java rename extractor/src/test/java/org/schabi/newpipe/extractor/utils/{ => cache}/ManifestCreatorCacheTest.java (96%) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java index b0ac2e14f7..9e04238bf4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/InfoItemsCollector.java @@ -78,7 +78,7 @@ public void reset() { * Add an error * @param error the error */ - protected void addError(final Exception error) { + public void addError(final Exception error) { errors.add(error); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java index 56af6b43eb..148ec92004 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java @@ -19,7 +19,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper; - +import org.schabi.newpipe.extractor.utils.cache.SoundCloudCommentsCache; +import org.schabi.newpipe.extractor.utils.cache.SoundCloudCommentsCache.CachedCommentInfo; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -37,7 +38,8 @@ public class SoundcloudCommentsExtractor extends CommentsExtractor { * and therefore the {@link SoundcloudCommentsInfoItemExtractor#replyCount} * of the last top level comment cannot be determined certainly. */ - @Nullable private JsonObject lastTopLevelComment; + private static final SoundCloudCommentsCache LAST_TOP_LEVEL_COMMENTS = + new SoundCloudCommentsCache(10); public SoundcloudCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) { @@ -100,7 +102,18 @@ public InfoItemsPage getPage(final Page page) } catch (final JsonParserException e) { throw new ParsingException("Could not parse json", e); } - collectCommentsFrom(collector, json, lastTopLevelComment); + + final CachedCommentInfo topLevelCommentElement = LAST_TOP_LEVEL_COMMENTS.get(getUrl()); + if (topLevelCommentElement == null) { + if (LAST_TOP_LEVEL_COMMENTS.isEmpty()) { + collector.addError(new RuntimeException( + "Could not get last top level comment. It has been removed from cache." + + " Increase the cache size to not loose any comments")); + } + collectCommentsFrom(collector, json, null); + } else { + collectCommentsFrom(collector, json, topLevelCommentElement); + } } if (hasNextPage) { @@ -111,54 +124,71 @@ public InfoItemsPage getPage(final Page page) } @Override - public void onFetchPage(@Nonnull final Downloader downloader) { } + public void onFetchPage(@Nonnull final Downloader downloader) { + } /** * Collect top level comments from a SoundCloud API response. - * @param collector the collector which collects the the top level comments - * @param json the JsonObject of the API response + * + * @param collector the collector which collects the the top level comments + * @param json the JsonObject of the API response * @param lastTopLevelComment the last top level comment from the previous page or {@code null} * if this method is run for the initial page. * @throws ParsingException */ private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector collector, @Nonnull final JsonObject json, - @Nullable final JsonObject lastTopLevelComment) + @Nullable final CachedCommentInfo lastTopLevelComment) throws ParsingException { final List extractors = new ArrayList<>(); final String url = getUrl(); - final JsonArray entries = json.getArray(COLLECTION); - /** - * The current top level comment. - */ + JsonObject currentTopLevelComment = null; + int currentTopLevelCommentIndex = 0; boolean isLastCommentReply = true; + boolean isFirstCommentReply = false; + boolean addedLastTopLevelComment = lastTopLevelComment == null; // Check whether the first comment in the list is a reply to the last top level comment // from the previous page if there was a previous page. if (lastTopLevelComment != null) { - final JsonObject firstComment = entries.getObject(0); - if (SoundcloudParsingHelper.isReplyTo(lastTopLevelComment, firstComment)) { - currentTopLevelComment = lastTopLevelComment; + final JsonObject firstComment = json.getArray(COLLECTION).getObject(0); + if (SoundcloudParsingHelper.isReplyTo(lastTopLevelComment.comment, firstComment)) { + currentTopLevelComment = lastTopLevelComment.comment; + isFirstCommentReply = true; + merge(json, lastTopLevelComment.json, lastTopLevelComment.index); } else { extractors.add(new SoundcloudCommentsInfoItemExtractor( - json, SoundcloudCommentsInfoItemExtractor.PREVIOUS_PAGE_INDEX, - firstComment, url, null)); + lastTopLevelComment.json, + lastTopLevelComment.index, + lastTopLevelComment.comment, url, null)); + addedLastTopLevelComment = true; } } + final JsonArray entries = json.getArray(COLLECTION); for (int i = 0; i < entries.size(); i++) { final JsonObject entry = entries.getObject(i); - // extract all top level comments - // The first comment is either a top level comment + // Extract all top level comments + // The first comment is a top level co // if it is not a reply to the last top level comment // - if (i == 0 && currentTopLevelComment == null - || (!SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) - && !SoundcloudParsingHelper.isReplyTo(currentTopLevelComment, entry))) { + if ((i == 0 && !isFirstCommentReply) + || ( + i != 0 && !SoundcloudParsingHelper.isReplyTo(entries.getObject(i - 1), entry) + && !SoundcloudParsingHelper.isReplyTo(currentTopLevelComment, entry))) { currentTopLevelComment = entry; + currentTopLevelCommentIndex = i; + if (!addedLastTopLevelComment) { + // There is a new top level comment. This also means that we can now determine + // the reply count and get all replies for the top level comment. + extractors.add(new SoundcloudCommentsInfoItemExtractor( + json, 0, lastTopLevelComment.comment, url, null)); + addedLastTopLevelComment = true; + } if (i == entries.size() - 1) { isLastCommentReply = false; - this.lastTopLevelComment = currentTopLevelComment; + LAST_TOP_LEVEL_COMMENTS.put(getUrl(), currentTopLevelComment, json, i); + // Do not collect the last comment if it is a top level comment // because it might have replies. // That is information we cannot get from the comment itself @@ -168,14 +198,17 @@ private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector colle break; } extractors.add(new SoundcloudCommentsInfoItemExtractor( - json, i, entry, url, lastTopLevelComment)); + json, i, entry, url, null)); } } if (isLastCommentReply) { // Do not collect the last top level comment if it has replies and the retrieved // comment list ends with a reply. We do not know whether the next page starts // with more replies to the last top level comment. - this.lastTopLevelComment = extractors.remove(extractors.size() - 1).item; + LAST_TOP_LEVEL_COMMENTS.put( + getUrl(), + extractors.remove(extractors.size() - 1).item, + json, currentTopLevelCommentIndex); } extractors.stream().forEach(collector::commit); @@ -183,11 +216,13 @@ private void collectCommentsFrom(@Nonnull final CommentsInfoItemsCollector colle /** * Collect replies to a top level comment from a SoundCloud API response. + * * @param collector the collector which collects the the replies - * @param json the SoundCloud API response - * @param id the comment's id for which the replies are collected - * @param url the corresponding page's URL - * @return + * @param json the SoundCloud API response + * @param id the comment's id for which the replies are collected + * @param url the corresponding page's URL + * @return {code true} if there might be more replies to the comment; + * {@code false} if there are definitely no more replies */ private boolean collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector collector, @Nonnull final JsonObject json, @@ -206,8 +241,8 @@ private boolean collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector col && SoundcloudParsingHelper.isReplyTo(originalComment, comment)) { collector.commit(new SoundcloudCommentsInfoItemExtractor( json, i, entries.getObject(i), url, originalComment)); - // There might be more replies to the originalComment, - // especially if the original comment is at the end of the list. + // There might be more replies to the originalComment + // if the original comment is at the end of the list. if (i == entries.size() - 1 && json.has(NEXT_HREF)) { moreReplies = true; } @@ -216,4 +251,17 @@ private boolean collectRepliesFrom(@Nonnull final CommentsInfoItemsCollector col return moreReplies; } + private void merge(@Nonnull final JsonObject target, @Nonnull final JsonObject subject, + final int index) { + final JsonArray targetArray = target.getArray(COLLECTION); + final JsonArray subjectArray = subject.getArray(COLLECTION); + final JsonArray newArray = new JsonArray( + targetArray.size() + subjectArray.size() - index - 1); + for (int i = index; i < subjectArray.size(); i++) { + newArray.add(subjectArray.getObject(i)); + } + newArray.addAll(targetArray); + target.put(COLLECTION, newArray); + } + } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java index 0dfa3edeee..78afff6fd5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsInfoItemExtractor.java @@ -1,5 +1,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors; +import static org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudCommentsExtractor.COLLECTION; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import com.grack.nanojson.JsonArray; @@ -59,6 +60,13 @@ public SoundcloudCommentsInfoItemExtractor(final JsonObject json, final int inde this(json, index, item, url, null); } + public void addInfoFromNextPage(@Nonnull final JsonArray newItems, final int itemCount) { + final JsonArray currentItems = this.json.getArray(COLLECTION); + for (int i = 0; i < itemCount; i++) { + currentItems.add(newItems.getObject(i)); + } + } + @Override public String getCommentId() { return Objects.toString(item.getLong("id"), null); @@ -75,7 +83,7 @@ public Description getCommentText() { // We need to do this manually. if (commentContent.startsWith("@")) { final String authorName = commentContent.split(" ", 2)[0].replace("@", ""); - final JsonArray comments = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + final JsonArray comments = json.getArray(COLLECTION); JsonObject author = null; for (int i = index - 1; i >= 0 && author == null; i--) { final JsonObject commentsAuthor = comments.getObject(i).getObject("user"); @@ -163,7 +171,7 @@ public Page getReplies() { if (topLevelComment == null) { // Loop through all comments which come after the original comment // to find its replies. - final JsonArray allItems = json.getArray(SoundcloudCommentsExtractor.COLLECTION); + final JsonArray allItems = json.getArray(COLLECTION); for (int i = index + 1; i < allItems.size(); i++) { if (SoundcloudParsingHelper.isReplyTo(item, allItems.getObject(i))) { replyCount++; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java index 46bd324204..fcf07d0937 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java @@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.Attr; import org.w3c.dom.DOMException; import org.w3c.dom.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java index 46e84df1db..9226d8d2fd 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeOtfDashManifestCreator.java @@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.schabi.newpipe.extractor.utils.Utils; import org.w3c.dom.DOMException; import org.w3c.dom.Document; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java index 3a5a7dd23d..5c23138f08 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubePostLiveStreamDvrDashManifestCreator.java @@ -15,7 +15,7 @@ import org.schabi.newpipe.extractor.downloader.Response; import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java index 0f69895bba..1c1e04c37d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/dashmanifestcreators/YoutubeProgressiveDashManifestCreator.java @@ -2,7 +2,7 @@ import org.schabi.newpipe.extractor.services.youtube.DeliveryType; import org.schabi.newpipe.extractor.services.youtube.ItagItem; -import org.schabi.newpipe.extractor.utils.ManifestCreatorCache; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java new file mode 100644 index 0000000000..6e8180e93f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/Cache.java @@ -0,0 +1,9 @@ +package org.schabi.newpipe.extractor.utils.cache; + +public interface Cache { + void put(K key, V value); + V get(K key); + int size(); + boolean isEmpty(); + void clear(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java similarity index 98% rename from extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java rename to extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java index ac12f83f95..149369c53e 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCache.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCache.java @@ -1,4 +1,6 @@ -package org.schabi.newpipe.extractor.utils; +package org.schabi.newpipe.extractor.utils.cache; + +import org.schabi.newpipe.extractor.utils.Pair; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java new file mode 100644 index 0000000000..5c367ce49f --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCache.java @@ -0,0 +1,74 @@ +package org.schabi.newpipe.extractor.utils.cache; + +import com.grack.nanojson.JsonObject; + +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * LRU cache which can contain a few items. + */ +public class SoundCloudCommentsCache { + + private final int maxSize; + private final Map store; + public SoundCloudCommentsCache(final int size) { + if (size < 1) { + throw new IllegalArgumentException("Size must be at least 1"); + } + store = new HashMap<>(size); + maxSize = size; + } + + public void put(@Nonnull final String key, @Nonnull final JsonObject comment, + @Nonnull final JsonObject json, final int index) { + if (store.size() == maxSize) { + store.remove( + store.entrySet().stream() + .reduce((a, b) -> a.getValue().lastHit < b.getValue().lastHit ? a : b) + .get().getKey()); + } + store.put(key, new CachedCommentInfo(comment, json, index)); + } + + @Nullable + public CachedCommentInfo get(final String key) { + final CachedCommentInfo result = store.get(key); + if (result == null) { + return null; + } + result.lastHit = System.nanoTime(); + return result; + } + + public int size() { + return store.size(); + } + + public boolean isEmpty() { + return store.isEmpty(); + } + + public void clear() { + store.clear(); + } + + public final class CachedCommentInfo { + @Nonnull public final JsonObject comment; + @Nonnull public final JsonObject json; + public final int index; + private long lastHit = System.nanoTime(); + + private CachedCommentInfo(@Nonnull final JsonObject comment, + @Nonnull final JsonObject json, + final int index) { + this.comment = comment; + this.json = json; + this.index = index; + } + } + +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java similarity index 96% rename from extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java rename to extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java index 83c5c1dfb1..a28d745ebd 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ManifestCreatorCacheTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/ManifestCreatorCacheTest.java @@ -1,6 +1,7 @@ -package org.schabi.newpipe.extractor.utils; +package org.schabi.newpipe.extractor.utils.cache; import org.junit.jupiter.api.Test; +import org.schabi.newpipe.extractor.utils.cache.ManifestCreatorCache; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java new file mode 100644 index 0000000000..bd985905a5 --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/cache/SoundCloudCommentsCacheTest.java @@ -0,0 +1,83 @@ +package org.schabi.newpipe.extractor.utils.cache; + +import com.grack.nanojson.JsonObject; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SoundCloudCommentsCacheTest { + @Test + void testInstantiation() { + assertThrows(RuntimeException.class, () -> new SoundCloudCommentsCache(-15)); + assertThrows(RuntimeException.class, () -> new SoundCloudCommentsCache(0)); + assertDoesNotThrow(() -> new SoundCloudCommentsCache(1)); + assertDoesNotThrow(() -> new SoundCloudCommentsCache(10)); + } + + @Test + void testSize() { + SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + assertEquals(0, cache.size()); + assertTrue(cache.isEmpty()); + cache.put("a", new JsonObject(), new JsonObject(), 1); + assertEquals(1, cache.size()); + cache.put("b", new JsonObject(), new JsonObject(), 1); + assertEquals(2, cache.size()); + cache.put("c", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.put("a", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.put("b", new JsonObject(), new JsonObject(), 1); + assertEquals(3, cache.size()); + cache.clear(); + assertEquals(0, cache.size()); + } + + @Test + void testLRUStrategy() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(4); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("2", new JsonObject(), new JsonObject(), 2); + cache.put("3", new JsonObject(), new JsonObject(), 3); + cache.put("4", new JsonObject(), new JsonObject(), 4); + cache.put("5", new JsonObject(), new JsonObject(), 5); + assertNull(cache.get("1")); + final SoundCloudCommentsCache.CachedCommentInfo cci = cache.get("2"); + assertNotNull(cci); + cache.put("6", new JsonObject(), new JsonObject(), 6); + assertNotNull(cache.get("2")); + assertNull(cache.get("3")); + cache.put("7", new JsonObject(), new JsonObject(), 7); + cache.put("8", new JsonObject(), new JsonObject(), 8); + cache.put("9", new JsonObject(), new JsonObject(), 9); + assertNull(cache.get("1")); + assertNull(cache.get("3")); + assertNull(cache.get("4")); + assertNull(cache.get("5")); + assertNotNull(cache.get("2")); + } + + @Test + void testStorage() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("1", new JsonObject(), new JsonObject(), 2); + assertEquals(2, cache.get("1").index); + cache.put("1", new JsonObject(), new JsonObject(), 3); + assertEquals(3, cache.get("1").index); + } + + @Test + void testClear() { + final SoundCloudCommentsCache cache = new SoundCloudCommentsCache(10); + cache.put("1", new JsonObject(), new JsonObject(), 1); + cache.put("2", new JsonObject(), new JsonObject(), 2); + cache.put("3", new JsonObject(), new JsonObject(), 3); + cache.put("4", new JsonObject(), new JsonObject(), 4); + cache.put("5", new JsonObject(), new JsonObject(), 5); + cache.clear(); + assertTrue(cache.isEmpty()); + assertEquals(0, cache.size()); + } + +}