Merge pull request #628 from litetex/fix-broken-yt-liked-comments

Fix broken yt likes in comments
This commit is contained in:
bopol 2021-05-29 11:04:11 +02:00 committed by GitHub
commit ff11c2df2a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 630 additions and 178 deletions

View file

@ -17,6 +17,7 @@ public class CommentsInfoItem extends InfoItem {
@Nullable @Nullable
private DateWrapper uploadDate; private DateWrapper uploadDate;
private int likeCount; private int likeCount;
private String textualLikeCount;
private boolean heartedByUploader; private boolean heartedByUploader;
private boolean pinned; private boolean pinned;
@ -89,6 +90,14 @@ public class CommentsInfoItem extends InfoItem {
this.likeCount = likeCount; this.likeCount = likeCount;
} }
public String getTextualLikeCount() {
return textualLikeCount;
}
public void setTextualLikeCount(String textualLikeCount) {
this.textualLikeCount = textualLikeCount;
}
public void setHeartedByUploader(boolean isHeartedByUploader) { public void setHeartedByUploader(boolean isHeartedByUploader) {
this.heartedByUploader = isHeartedByUploader; this.heartedByUploader = isHeartedByUploader;
} }

View file

@ -3,30 +3,50 @@ package org.schabi.newpipe.extractor.comments;
import org.schabi.newpipe.extractor.InfoItemExtractor; import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable; import javax.annotation.Nullable;
public interface CommentsInfoItemExtractor extends InfoItemExtractor { public interface CommentsInfoItemExtractor extends InfoItemExtractor {
/** /**
* Return the like count of the comment, or -1 if it's unavailable * Return the like count of the comment, or -1 if it's unavailable<br/>
*
* NOTE: Currently only implemented for YT {@link YoutubeCommentsInfoItemExtractor#getLikeCount()}
* with limitations (only approximate like count is returned)
* *
* @see StreamExtractor#getLikeCount() * @see StreamExtractor#getLikeCount()
*/ */
int getLikeCount() throws ParsingException; default int getLikeCount() throws ParsingException {
return -1;
}
/**
* The unmodified like count given by the service<br/>
*
* It may be language dependent
*/
default String getTextualLikeCount() throws ParsingException {
return Utils.EMPTY_STRING;
}
/** /**
* The text of the comment * The text of the comment
*/ */
String getCommentText() throws ParsingException; default String getCommentText() throws ParsingException {
return Utils.EMPTY_STRING;
}
/** /**
* The upload date given by the service, unmodified * The upload date given by the service, unmodified
* *
* @see StreamExtractor#getTextualUploadDate() * @see StreamExtractor#getTextualUploadDate()
*/ */
String getTextualUploadDate() throws ParsingException; default String getTextualUploadDate() throws ParsingException {
return Utils.EMPTY_STRING;
}
/** /**
* The upload date wrapped with DateWrapper class * The upload date wrapped with DateWrapper class
@ -34,28 +54,44 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
* @see StreamExtractor#getUploadDate() * @see StreamExtractor#getUploadDate()
*/ */
@Nullable @Nullable
DateWrapper getUploadDate() throws ParsingException; default DateWrapper getUploadDate() throws ParsingException {
return null;
}
String getCommentId() throws ParsingException; default String getCommentId() throws ParsingException {
return Utils.EMPTY_STRING;
}
String getUploaderUrl() throws ParsingException; default String getUploaderUrl() throws ParsingException {
return Utils.EMPTY_STRING;
}
String getUploaderName() throws ParsingException; default String getUploaderName() throws ParsingException {
return Utils.EMPTY_STRING;
}
String getUploaderAvatarUrl() throws ParsingException; default String getUploaderAvatarUrl() throws ParsingException {
return Utils.EMPTY_STRING;
}
/** /**
* Whether the comment has been hearted by the uploader * Whether the comment has been hearted by the uploader
*/ */
boolean isHeartedByUploader() throws ParsingException; default boolean isHeartedByUploader() throws ParsingException {
return false;
}
/** /**
* Whether the comment is pinned * Whether the comment is pinned
*/ */
boolean isPinned() throws ParsingException; default boolean isPinned() throws ParsingException {
return false;
}
/** /**
* Whether the uploader is verified by the service * Whether the uploader is verified by the service
*/ */
boolean isUploaderVerified() throws ParsingException; default boolean isUploaderVerified() throws ParsingException {
return false;
}
} }

View file

@ -6,7 +6,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Vector;
public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> { public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoItem, CommentsInfoItemExtractor> {
@ -65,6 +64,11 @@ public class CommentsInfoItemsCollector extends InfoItemsCollector<CommentsInfoI
} catch (Exception e) { } catch (Exception e) {
addError(e); addError(e);
} }
try {
resultItem.setTextualLikeCount(extractor.getTextualLikeCount());
} catch (Exception e) {
addError(e);
}
try { try {
resultItem.setThumbnailUrl(extractor.getThumbnailUrl()); resultItem.setThumbnailUrl(extractor.getThumbnailUrl());
} catch (Exception e) { } catch (Exception e) {

View file

@ -3,9 +3,6 @@ package org.schabi.newpipe.extractor.services.bandcamp.extractors;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import javax.annotation.Nullable;
public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor { public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
@ -32,39 +29,11 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return writing.getElementsByClass("thumb").attr("src"); return writing.getElementsByClass("thumb").attr("src");
} }
@Override
public int getLikeCount() {
return -1;
}
@Override @Override
public String getCommentText() { public String getCommentText() {
return writing.getElementsByClass("text").first().ownText(); return writing.getElementsByClass("text").first().ownText();
} }
@Override
public String getTextualUploadDate() {
return "";
}
@Nullable
@Override
public DateWrapper getUploadDate() {
return null;
}
@Override
public String getCommentId() {
return "";
}
@Override
public String getUploaderUrl() {
//return writing.getElementsByClass("name").attr("href");
// Fan links cannot be opened
return "";
}
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
return writing.getElementsByClass("name").first().text(); return writing.getElementsByClass("name").first().text();
@ -74,19 +43,4 @@ public class BandcampCommentsInfoItemExtractor implements CommentsInfoItemExtrac
public String getUploaderAvatarUrl() { public String getUploaderAvatarUrl() {
return writing.getElementsByClass("thumb").attr("src"); return writing.getElementsByClass("thumb").attr("src");
} }
@Override
public boolean isHeartedByUploader() {
return false;
}
@Override
public boolean isPinned() {
return false;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
} }

View file

@ -57,11 +57,6 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate)); return new DateWrapper(PeertubeParsingHelper.parseDateFrom(textualUploadDate));
} }
@Override
public int getLikeCount() {
return -1;
}
@Override @Override
public String getCommentText() throws ParsingException { public String getCommentText() throws ParsingException {
final String htmlText = JsonUtils.getString(item, "text"); final String htmlText = JsonUtils.getString(item, "text");
@ -89,21 +84,6 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return baseUrl + value; return baseUrl + value;
} }
@Override
public boolean isHeartedByUploader() throws ParsingException {
return false;
}
@Override
public boolean isPinned() throws ParsingException {
return false;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override @Override
public String getUploaderName() throws ParsingException { public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host"); return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");

View file

@ -38,16 +38,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
return json.getObject("user").getString("avatar_url"); return json.getObject("user").getString("avatar_url");
} }
@Override
public boolean isHeartedByUploader() throws ParsingException {
return false;
}
@Override
public boolean isPinned() throws ParsingException {
return false;
}
@Override @Override
public boolean isUploaderVerified() throws ParsingException { public boolean isUploaderVerified() throws ParsingException {
return json.getObject("user").getBoolean("verified"); return json.getObject("user").getBoolean("verified");
@ -69,11 +59,6 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate())); return new DateWrapper(SoundcloudParsingHelper.parseDateFrom(getTextualUploadDate()));
} }
@Override
public int getLikeCount() {
return -1;
}
@Override @Override
public String getName() throws ParsingException { public String getName() throws ParsingException {
return json.getObject("user").getString("permalink"); return json.getObject("user").getString("permalink");

View file

@ -2,6 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject; import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor; import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException; import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.localization.DateWrapper;
@ -70,12 +71,70 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
} }
} }
/**
* @implNote The method is parsing internally a localized string.<br/>
* <ul>
* <li>
* More than >1k likes will result in an inaccurate number
* </li>
* <li>
* This will fail for other languages than English.
* However as long as the Extractor only uses "en-GB"
* (as seen in {@link org.schabi.newpipe.extractor.services.youtube.YoutubeService#SUPPORTED_LANGUAGES})
* everything will work fine.
* </li>
* </ul>
* <br/>
* Consider using {@link #getTextualLikeCount()}
*/
@Override @Override
public int getLikeCount() throws ParsingException { public int getLikeCount() throws ParsingException {
// This may return a language dependent version, e.g. in German: 3,3 Mio
final String textualLikeCount = getTextualLikeCount();
try { try {
return json.getInt("likeCount"); if (Utils.isBlank(textualLikeCount)) {
return 0;
}
return (int) Utils.mixedNumberWordToLong(textualLikeCount);
} catch (Exception e) { } catch (Exception e) {
throw new ParsingException("Could not get like count", e); throw new ParsingException("Unexpected error while converting textual like count to like count", e);
}
}
@Override
public String getTextualLikeCount() throws ParsingException {
/*
* Example results as of 2021-05-20:
* Language = English
* 3.3M
* 48K
* 1.4K
* 270K
* 19
* 6
*
* Language = German
* 3,3 Mio
* 48.189
* 1419
* 270.984
* 19
* 6
*/
try {
// If a comment has no likes voteCount is not set
if (!json.has("voteCount")) {
return EMPTY_STRING;
}
final JsonObject voteCountObj = JsonUtils.getObject(json, "voteCount");
if (voteCountObj.isEmpty()) {
return EMPTY_STRING;
}
return getTextFromObject(voteCountObj);
} catch (Exception e) {
throw new ParsingException("Could not get vote count", e);
} }
} }

View file

@ -13,10 +13,10 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import static org.junit.Assert.*; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class BandcampCommentsExtractorTest { public class BandcampCommentsExtractorTest {
@ -47,6 +47,7 @@ public class BandcampCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount()); assertEquals(-1, c.getLikeCount());
assertTrue(Utils.isBlank(c.getTextualLikeCount()));
} }
} }
} }

View file

@ -75,7 +75,8 @@ public class PeertubeCommentsExtractorTest {
assertFalse(Utils.isBlank(c.getTextualUploadDate())); assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1); assertEquals(-1, c.getLikeCount());
assertTrue(Utils.isBlank(c.getTextualLikeCount()));
} }
} }

View file

@ -9,12 +9,14 @@ import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.comments.CommentsInfo; import org.schabi.newpipe.extractor.comments.CommentsInfo;
import org.schabi.newpipe.extractor.comments.CommentsInfoItem; import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.DefaultTests; import org.schabi.newpipe.extractor.services.DefaultTests;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Locale;
import java.util.Random; import java.util.Random;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
@ -98,7 +100,7 @@ public class YoutubeCommentsExtractorTest {
assertNotNull(c.getUploadDate()); assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0); assertTrue(c.getLikeCount() >= 0);
} }
} }
@ -148,7 +150,7 @@ public class YoutubeCommentsExtractorTest {
assertNotNull(c.getUploadDate()); assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0); assertTrue(c.getLikeCount() >= 0);
if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text if (c.getCommentId().equals("Ugga_h1-EXdHB3gCoAEC")) { // comment without text
assertTrue(Utils.isBlank(c.getCommentText())); assertTrue(Utils.isBlank(c.getCommentText()));
} else { } else {
@ -191,7 +193,7 @@ public class YoutubeCommentsExtractorTest {
assertNotNull(c.getUploadDate()); assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0); assertTrue(c.getLikeCount() >= 0);
assertFalse(Utils.isBlank(c.getCommentText())); assertFalse(Utils.isBlank(c.getCommentText()));
if (c.isHeartedByUploader()) { if (c.isHeartedByUploader()) {
heartedByUploader = true; heartedByUploader = true;
@ -232,11 +234,76 @@ public class YoutubeCommentsExtractorTest {
assertNotNull(c.getUploadDate()); assertNotNull(c.getUploadDate());
assertFalse(Utils.isBlank(c.getThumbnailUrl())); assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl())); assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() < 0); assertTrue(c.getLikeCount() >= 0);
assertFalse(Utils.isBlank(c.getCommentText())); assertFalse(Utils.isBlank(c.getCommentText()));
} }
assertTrue("First comment isn't pinned", comments.getItems().get(0).isPinned()); assertTrue("First comment isn't pinned", comments.getItems().get(0).isPinned());
} }
} }
/**
* Checks if the likes/votes are handled correctly<br/>
* A pinned comment with >15K likes is used for the test
*/
public static class LikesVotes {
private final static String url = "https://www.youtube.com/watch?v=QqsLTNkzvaY";
private static YoutubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "likes"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
@Test
public void testGetCommentsFirst() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
CommentsInfoItem pinnedComment = comments.getItems().get(0);
assertTrue("First comment isn't pinned", pinnedComment.isPinned());
assertTrue("The first pinned comment has no likes", pinnedComment.getLikeCount() > 0);
assertTrue("The first pinned comment has no vote count", !Utils.isBlank(pinnedComment.getTextualLikeCount()));
}
}
/**
* Checks if the vote count works localized<br/>
* A pinned comment with >15K likes is used for the test
*/
public static class LocalizedVoteCount {
private final static String url = "https://www.youtube.com/watch?v=QqsLTNkzvaY";
private static YoutubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localized_vote_count"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
// Force non english local here
extractor.forceLocalization(Localization.fromLocale(Locale.GERMANY));
extractor.fetchPage();
}
@Test
public void testGetCommentsFirst() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
DefaultTests.defaultTestListOfItems(YouTube, comments.getItems(), comments.getErrors());
CommentsInfoItem pinnedComment = comments.getItems().get(0);
assertTrue("First comment isn't pinned", pinnedComment.isPinned());
assertTrue("The first pinned comment has no vote count", !Utils.isBlank(pinnedComment.getTextualLikeCount()));
}
}
} }