CypherpunkSamurai
/
NewPipeExtractor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor frames extraction

This commit is contained in:
Vasiliy 2019-09-10 19:38:51 +03:00
parent a6c94c7a9d
commit f084cfec24
No known key found for this signature in database
GPG Key ID: 9F74C4D2874D7523
5 changed files with 140 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View File

@ -1036,28 +1036,60 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}; };
} }
@Nullable @Nonnull
public StreamFrames getFrames() { @Override
try { public List<Frameset> getFrames() throws ExtractionException {
final String script = doc.select("#player-api").first().siblingElements().select("script").html(); try {
int p = script.indexOf("ytplayer.config"); final String script = doc.select("#player-api").first().siblingElements().select("script").html();
if (p == -1) { int p = script.indexOf("ytplayer.config");
return null; if (p == -1) {
} return Collections.emptyList();
p = script.indexOf('{', p); }
int e = script.indexOf("ytplayer.load", p); p = script.indexOf('{', p);
if (e == -1) { int e = script.indexOf("ytplayer.load", p);
return null; if (e == -1) {
} return Collections.emptyList();
JsonObject jo = JsonParser.object().from(script.substring(p, e - 1)); }
final String resp = jo.getObject("args").getString("player_response"); JsonObject jo = JsonParser.object().from(script.substring(p, e - 1));
jo = JsonParser.object().from(resp); final String resp = jo.getObject("args").getString("player_response");
final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|"); jo = JsonParser.object().from(resp);
final String url = spec[0]; final String[] spec = jo.getObject("storyboards").getObject("playerStoryboardSpecRenderer").getString("spec").split("\\|");
final List<String> opts = Arrays.asList(spec).subList(1, spec.length); final String url = spec[0];
return new StreamFrames(url, opts); final ArrayList<Frameset> result = new ArrayList<>(spec.length - 1);
} catch (Exception e) { for (int i = 1; i < spec.length; ++i) {
return null; final String[] parts = spec[i].split("#");
} if (parts.length != 8) {
} continue;
}
final int frameWidth = Integer.parseInt(parts[0]);
final int frameHeight = Integer.parseInt(parts[1]);
final int totalCount = Integer.parseInt(parts[2]);
final int framesPerPageX = Integer.parseInt(parts[3]);
final int framesPerPageY = Integer.parseInt(parts[4]);
final String baseUrl = url.replace("$L", String.valueOf(i - 1)).replace("$N", parts[6]) + "&sigh=" + parts[7];
final List<String> urls;
if (baseUrl.contains("$M")) {
final int totalPages = (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
urls = new ArrayList<>(totalPages);
for (int j = 0; j < totalPages; j++) {
urls.add(baseUrl.replace("$M", String.valueOf(j)));
}
} else {
urls = Collections.singletonList(baseUrl);
}
result.add(new Frameset(
urls,
frameWidth,
frameHeight,
totalCount,
framesPerPageX,
framesPerPageY
));
}
result.trimToSize();
return result;
} catch (Exception e) {
throw new ExtractionException(e);
}
}
} }

63
extractor/src/main/java/org/schabi/newpipe/extractor/stream/Frameset.java Normal file
View File

@ -0,0 +1,63 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.util.Collection;
import java.util.List;
public final class Frameset {
private List<String> urls;
private int frameWidth;
private int frameHeight;
private int totalCount;
private int framesPerPageX;
private int framesPerPageY;
public Frameset(List<String> urls, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
this.urls = urls;
this.totalCount = totalCount;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
this.framesPerPageY = framesPerPageY;
}
public List<String> getUrls() {
return urls;
}
/**
* @return total count of frames
*/
public int getTotalCount() {
return totalCount;
}
/**
* @return maximum frames count by x
*/
public int getFramesPerPageX() {
return framesPerPageX;
}
/**
* @return maximum frames count by y
*/
public int getFramesPerPageY() {
return framesPerPageY;
}
/**
* @return width of a one frame, in pixels
*/
public int getFrameWidth() {
return frameWidth;
}
/**
* @return height of a one frame, in pixels
*/
public int getFrameHeight() {
return frameHeight;
}
}

12
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamExtractor.java
View File

@ -30,7 +30,10 @@ import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Parser; import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull; import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List; import java.util.List;
/** /**
@ -255,6 +258,15 @@ public abstract class StreamExtractor extends Extractor {
*/ */
public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException; public abstract StreamInfoItemsCollector getRelatedStreams() throws IOException, ExtractionException;
/**
* Should return a list of frames
* @return
*/
@Nonnull
public List<Frameset> getFrames() throws IOException, ExtractionException {
return Collections.emptyList();
}
/** /**
* Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block) * Should analyse the webpage's document and extracts any error message there might be. (e.g. GEMA block)
* *

113
extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamFrames.java
View File

@ -1,113 +0,0 @@
package org.schabi.newpipe.extractor.stream;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.List;
public class StreamFrames {
private final List<Frameset> frames;
public StreamFrames(String baseUrl, List<String> params) {
frames = new ArrayList<>(params.size());
for (int i = 0; i < params.size(); i++) {
String param = params.get(i);
final String[] parts = param.split("#");
frames.add(new Frameset(
baseUrl.replace("$L", String.valueOf(i)).replace("$N", parts[6]) + "&sigh=" + parts[7],
Integer.parseInt(parts[0]),
Integer.parseInt(parts[1]),
Integer.parseInt(parts[2]),
Integer.parseInt(parts[3]),
Integer.parseInt(parts[4])
));
}
}
public int getVariantsCount() {
return frames.size();
}
public Frameset getVariant(int index) {
return frames.get(index);
}
@Nullable
public Frameset getDefaultVariant() {
for (final Frameset f : frames) {
if (f.getUrl().contains("default.jpg")) {
return f;
}
}
return null;
}
public static class Frameset {
private String url;
private int frameWidth;
private int frameHeight;
private int totalCount;
private int framesPerPageX;
private int framesPerPageY;
private Frameset(String url, int frameWidth, int frameHeight, int totalCount, int framesPerPageX, int framesPerPageY) {
this.url = url;
this.totalCount = totalCount;
this.frameWidth = frameWidth;
this.frameHeight = frameHeight;
this.framesPerPageX = framesPerPageX;
this.framesPerPageY = framesPerPageY;
}
public String getUrl() {
return url;
}
public String getUrl(int page) {
return url.replace("$M", String.valueOf(page));
}
public int getTotalPages() {
if (!url.contains("$M")) {
return 0;
}
return (int) Math.ceil(totalCount / (double) (framesPerPageX * framesPerPageY));
}
/**
* @return total count of frames
*/
public int getTotalCount() {
return totalCount;
}
/**
* @return maximum frames count by x
*/
public int getFramesPerPageX() {
return framesPerPageX;
}
/**
* @return maximum frames count by y
*/
public int getFramesPerPageY() {
return framesPerPageY;
}
/**
* @return width of a one frame, in pixels
*/
public int getFrameWidth() {
return frameWidth;
}
/**
* @return height of a one frame, in pixels
*/
public int getFrameHeight() {
return frameHeight;
}
}
}

17
extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractorDefaultTest.java
View File

@ -3,6 +3,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import org.schabi.newpipe.Downloader; import org.schabi.newpipe.Downloader;
import org.schabi.newpipe.extractor.ExtractorAsserts;
import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.exceptions.ExtractionException;
@ -13,6 +14,7 @@ import org.schabi.newpipe.extractor.utils.Localization;
import org.schabi.newpipe.extractor.utils.Utils; import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl; import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
@ -244,15 +246,14 @@ public class YoutubeStreamExtractorDefaultTest {
} }
@Test @Test
public void testGetFrames() { public void testGetFrames() throws ExtractionException {
final StreamFrames frames = extractor.getFrames(); final List<Frameset> frames = extractor.getFrames();
assertNotNull(frames); assertNotNull(frames);
assertNotNull(frames.getDefaultVariant()); assertFalse(frames.isEmpty());
for (int i=0;i<frames.getVariantsCount();i++) { for (final Frameset f : frames) {
final StreamFrames.Frameset frameset = frames.getVariant(i); for (final String url : f.getUrls()) {
final int pages = frameset.getTotalPages(); ExtractorAsserts.assertIsValidUrl(url);
final String url = pages == 0 ? frameset.getUrl() : frameset.getUrl(pages - 1); }
assertNotNull(url);
} }
} }
} }