Class Http

src/main/java/com/rarchives/ripme/utils/Http.java:32–275

Wrapper around the Jsoup connection methods. Benefit is retry logic.

 * Benefit is retry logic.
 */
public class Http {

    private static final int TIMEOUT = Utils.getConfigInteger("page.timeout", 5 * 1000);
    private static final Logger logger = LogManager.getLogger(Http.class);

    private int retries;
    private int retrySleep = 0;
    private final String url;
    private Connection connection;

    // Constructors
    public Http(String url) {
        this.url = url;
        defaultSettings();
    }

    private Http(URL url) {
        this.url = url.toExternalForm();
        defaultSettings();
    }

    public static Http url(String url) {
        return new Http(url);
    }

    public static Http url(URL url) {
        return new Http(url);
    }

    private void defaultSettings() {
        this.retries = Utils.getConfigInteger("download.retries", 3);
        this.retrySleep = Utils.getConfigInteger("download.retry.sleep", 5000);
        connection = Jsoup.connect(this.url);
        connection.userAgent(AbstractRipper.USER_AGENT);
        connection.method(Method.GET);
        connection.timeout(TIMEOUT);
        connection.maxBodySize(0);

        // Extract cookies from config entry:
        // Example config entry:
        // cookies.reddit.com = reddit_session=<value>; other_cookie=<value>
        connection.cookies(cookiesForURL(this.url));
    }

    private Map<String, String> cookiesForURL(String u) {
        Map<String, String> cookiesParsed = new HashMap<>();

        String cookieDomain = "";
        try {
            URL parsed = new URI(u).toURL();
            String cookieStr = "";

            String[] parts = parsed.getHost().split("\\.");

            // if url is www.reddit.com, we should also use cookies from reddit.com;
            // this rule is applied for all subdomains (for all rippers); e.g. also
            // old.reddit.com, new.reddit.com
            while (parts.length > 1) {