Merge pull request #349 from wb9688/okhttp

Use OkHttp for tests like in NewPipe

extractor/build.gradle

@@ -8,4 +8,5 @@ dependencies {
     implementation 'org.nibor.autolink:autolink:0.10.0'
 
     testImplementation 'junit:junit:4.13'
+    testImplementation "com.squareup.okhttp3:okhttp:3.12.11"
 }

extractor/src/test/java/org/schabi/newpipe/DownloaderTestImpl.java

@@ -4,123 +4,97 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Request;
 import org.schabi.newpipe.extractor.downloader.Response;
 import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
-import org.schabi.newpipe.extractor.localization.Localization;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import javax.net.ssl.HttpsURLConnection;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.List;
-import java.util.Map;
 
-public class DownloaderTestImpl extends Downloader {
+import okhttp3.OkHttpClient;
+import okhttp3.RequestBody;
+import okhttp3.ResponseBody;
 
-    private static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0";
+public final class DownloaderTestImpl extends Downloader {
-    private static final String DEFAULT_HTTP_ACCEPT_LANGUAGE = "en";
+    private static final String USER_AGENT
+            = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:68.0) Gecko/20100101 Firefox/68.0";
+    private static DownloaderTestImpl instance;
+    private OkHttpClient client;
 
-    private static DownloaderTestImpl instance = null;
+    private DownloaderTestImpl(final OkHttpClient.Builder builder) {
+        this.client = builder.readTimeout(30, TimeUnit.SECONDS).build();
+    }
 
-    private DownloaderTestImpl() {
+    /**
+     * It's recommended to call exactly once in the entire lifetime of the application.
+     *
+     * @param builder if null, default builder will be used
+     * @return a new instance of {@link DownloaderTestImpl}
+     */
+    public static DownloaderTestImpl init(@Nullable final OkHttpClient.Builder builder) {
+        instance = new DownloaderTestImpl(
+                builder != null ? builder : new OkHttpClient.Builder());
+        return instance;
     }
 
     public static DownloaderTestImpl getInstance() {
         if (instance == null) {
-            synchronized (DownloaderTestImpl.class) {
+            init(null);
-                if (instance == null) {
-                    instance = new DownloaderTestImpl();
-                }
-            }
         }
         return instance;
     }
 
-    private void setDefaultHeaders(URLConnection connection) {
-        connection.setRequestProperty("User-Agent", USER_AGENT);
-        connection.setRequestProperty("Accept-Language", DEFAULT_HTTP_ACCEPT_LANGUAGE);
-    }
-
     @Override
-    public Response execute(@Nonnull Request request) throws IOException, ReCaptchaException {
+    public Response execute(@Nonnull final Request request)
+            throws IOException, ReCaptchaException {
         final String httpMethod = request.httpMethod();
         final String url = request.url();
         final Map<String, List<String>> headers = request.headers();
-        @Nullable final byte[] dataToSend = request.dataToSend();
+        final byte[] dataToSend = request.dataToSend();
-        @Nullable final Localization localization = request.localization();
 
-        final HttpsURLConnection connection = (HttpsURLConnection) new URL(url).openConnection();
+        RequestBody requestBody = null;
+        if (dataToSend != null) {
+            requestBody = RequestBody.create(null, dataToSend);
+        }
 
-        connection.setConnectTimeout(30 * 1000); // 30s
+        final okhttp3.Request.Builder requestBuilder = new okhttp3.Request.Builder()
-        connection.setReadTimeout(30 * 1000); // 30s
+                .method(httpMethod, requestBody).url(url)
-        connection.setRequestMethod(httpMethod);
+                .addHeader("User-Agent", USER_AGENT);
-
-        setDefaultHeaders(connection);
 
         for (Map.Entry<String, List<String>> pair : headers.entrySet()) {
             final String headerName = pair.getKey();
             final List<String> headerValueList = pair.getValue();
 
             if (headerValueList.size() > 1) {
-                connection.setRequestProperty(headerName, null);
+                requestBuilder.removeHeader(headerName);
                 for (String headerValue : headerValueList) {
-                    connection.addRequestProperty(headerName, headerValue);
+                    requestBuilder.addHeader(headerName, headerValue);
                 }
             } else if (headerValueList.size() == 1) {
-                connection.setRequestProperty(headerName, headerValueList.get(0));
+                requestBuilder.header(headerName, headerValueList.get(0));
-            }
             }
 
-        @Nullable OutputStream outputStream = null;
-        @Nullable InputStreamReader input = null;
-        try {
-            if (dataToSend != null && dataToSend.length > 0) {
-                connection.setDoOutput(true);
-                connection.setRequestProperty("Content-Length", dataToSend.length + "");
-                outputStream = connection.getOutputStream();
-                outputStream.write(dataToSend);
         }
 
-            final InputStream inputStream = connection.getInputStream();
+        final okhttp3.Response response = client.newCall(requestBuilder.build()).execute();
-            final StringBuilder response = new StringBuilder();
 
-            // Not passing any charset for decoding here... something to keep in mind.
+        if (response.code() == 429) {
-            input = new InputStreamReader(inputStream);
+            response.close();
 
-            int readCount;
-            char[] buffer = new char[32 * 1024];
-            while ((readCount = input.read(buffer)) != -1) {
-                response.append(buffer, 0, readCount);
-            }
-
-            final int responseCode = connection.getResponseCode();
-            final String responseMessage = connection.getResponseMessage();
-            final Map<String, List<String>> responseHeaders = connection.getHeaderFields();
-            final String latestUrl = connection.getURL().toString();
-
-            return new Response(responseCode, responseMessage, responseHeaders, response.toString(), latestUrl);
-        } catch (Exception e) {
-            final int responseCode = connection.getResponseCode();
-
-            /*
-             * HTTP 429 == Too Many Request
-             * Receive from Youtube.com = ReCaptcha challenge request
-             * See : https://github.com/rg3/youtube-dl/issues/5138
-             */
-            if (responseCode == 429) {
             throw new ReCaptchaException("reCaptcha Challenge requested", url);
-            } else if (responseCode != -1) {
-                final String latestUrl = connection.getURL().toString();
-                return new Response(responseCode, connection.getResponseMessage(), connection.getHeaderFields(), null, latestUrl);
         }
 
-            throw new IOException("Error occurred while fetching the content", e);
+        final ResponseBody body = response.body();
-        } finally {
+        String responseBodyToReturn = null;
-            if (outputStream != null) outputStream.close();
+
-            if (input != null) input.close();
+        if (body != null) {
+            responseBodyToReturn = body.string();
         }
+
+        final String latestUrl = response.request().url().toString();
+        return new Response(response.code(), response.message(), response.headers().toMultimap(),
+                responseBodyToReturn, latestUrl);
     }
 }