From cfd4bad1ef700da1e029654d648e709b72b1d2aa Mon Sep 17 00:00:00 2001
From: ZapperDJ <zapperdj@gmail.com>
Date: Thu, 3 Nov 2022 17:44:32 +0100
Subject: [PATCH 01/66] Add script to unsave posts

---
 scripts/unsaveposts.py | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 scripts/unsaveposts.py

diff --git a/scripts/unsaveposts.py b/scripts/unsaveposts.py
new file mode 100644
index 0000000..c332520
--- /dev/null
+++ b/scripts/unsaveposts.py
@@ -0,0 +1,40 @@
+#! /usr/bin/env python3.9
+'''
+This script takes a list of submission IDs from a file named "successfulids" created with the 
+"extract_successful_ids.sh" script and unsaves them from your account. To make it work you must
+fill in the username and password fields below. Make sure you keep the quotes around the fields.
+You'll need to make a "user script" in your reddit profile to run this.
+Go to https://old.reddit.com/prefs/apps/
+Click on "Develop an app" at the bottom.
+Make sure you select a "script" not a "web app."
+Give it a random name. Doesn't matter.
+You need to fill in the "Redirect URI" field with something so go ahead and put 127.0.0.0 in there.
+Save it.
+The client ID is the 14 character string under the name you gave your script.
+It'll look like a bunch of random characters like this: pspYLwDoci9z_A
+The client secret is the longer string next to "secret".
+Replace those two fields below. Again keep the quotes around the fields.
+'''
+
+import praw
+
+try:
+    r= praw.Reddit(
+        client_id="CLIENTID",
+        client_secret="CLIENTSECRET",
+        password="USERPASSWORD",
+        user_agent="Unsave Posts",
+        username="USERNAME",
+    )
+
+    with open("successfulids", "r") as f:
+        for item in f:
+            r.submission(id = item.strip()).unsave()
+
+except:
+    print("Something went wrong. Did you install PRAW? Did you change the user login fields?")
+
+
+else:
+    print("Done! Thanks for playing!")
+

From 002a2dac4387fd936a706d386efcc6bce1ba69f3 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 15:20:03 +1000
Subject: [PATCH 02/66] Add line length to isort config

---
 tox.ini | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index e5dce99..a5b8495 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,4 +13,5 @@ commands =
 
 [isort]
 profile = black
-multi_line_output = 3
\ No newline at end of file
+multi_line_output = 3
+line_length = 120

From c4f636c388676314821b3449d875c3ff9b4c9acb Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 15:20:32 +1000
Subject: [PATCH 03/66] Fix import formatting

---
 bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
index 6109b7a..900c8e9 100644
--- a/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
+++ b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
@@ -9,9 +9,7 @@ from praw.models import Submission
 from bdfr.exceptions import NotADownloadableLinkError
 from bdfr.resource import Resource
 from bdfr.site_authenticator import SiteAuthenticator
-from bdfr.site_downloaders.fallback_downloaders.fallback_downloader import (
-    BaseFallbackDownloader,
-)
+from bdfr.site_downloaders.fallback_downloaders.fallback_downloader import BaseFallbackDownloader
 from bdfr.site_downloaders.youtube import Youtube
 
 logger = logging.getLogger(__name__)

From 82230a97bcc21f69ffa03a0a0183b55aa952f9e1 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 15:28:53 +1000
Subject: [PATCH 04/66] Add formatting check option

---
 tox.ini | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tox.ini b/tox.ini
index a5b8495..672927b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,7 @@
 [tox]
 envlist =
     format
+    format_check
 
 [testenv:format]
 deps =
@@ -11,6 +12,15 @@ commands =
     isort bdfr tests
     black bdfr tests --line-length 120
 
+[testenv:format_check]
+deps =
+    isort
+    black
+skip_install = True
+commands =
+    isort bdfr tests --check
+    black bdfr tests --line-length 120 --check
+
 [isort]
 profile = black
 multi_line_output = 3

From 9c3c5436b57a6528f283efbd67777f3b5e0650b2 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 15:49:41 +1000
Subject: [PATCH 05/66] Add formatting check option for code

---
 .github/workflows/formatting_check.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 .github/workflows/formatting_check.yml

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
new file mode 100644
index 0000000..498d6af
--- /dev/null
+++ b/.github/workflows/formatting_check.yml
@@ -0,0 +1,11 @@
+name: formatting_check
+run-name: Check code formatting
+on: pull_request
+jobs:
+  formatting_check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: paolorechia/pox@v1.0.1
+        with:
+          tox_env: "format_check"
\ No newline at end of file

From ee095d4814a2a8459e9edc41295c2df1fa63d10e Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 15:50:35 +1000
Subject: [PATCH 06/66] Expand action scope

---
 .github/workflows/formatting_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index 498d6af..8a04fc2 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -1,6 +1,6 @@
 name: formatting_check
 run-name: Check code formatting
-on: pull_request
+on: [push, pull_request]
 jobs:
   formatting_check:
     runs-on: ubuntu-latest

From 5427ceb29a7ba97f1507935937b4271bd80b5cb9 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:34:23 +1000
Subject: [PATCH 07/66] Add markdown linter to format check

---
 .github/workflows/formatting_check.yml | 1 +
 .markdown_style.rb                     | 3 +++
 tox.ini                                | 2 ++
 3 files changed, 6 insertions(+)
 create mode 100644 .markdown_style.rb

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index 8a04fc2..4941d83 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -7,5 +7,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: paolorechia/pox@v1.0.1
+      - uses: actionshub/markdownlint@main
         with:
           tox_env: "format_check"
\ No newline at end of file
diff --git a/.markdown_style.rb b/.markdown_style.rb
new file mode 100644
index 0000000..61f127b
--- /dev/null
+++ b/.markdown_style.rb
@@ -0,0 +1,3 @@
+all
+exclude_tag :line_length
+rule 'MD007', :indent => 4
diff --git a/tox.ini b/tox.ini
index 672927b..88df732 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,9 +17,11 @@ deps =
     isort
     black
 skip_install = True
+allowlist_externals = mdl
 commands =
     isort bdfr tests --check
     black bdfr tests --line-length 120 --check
+    mdl README.md docs/ -s .markdown_style.rb
 
 [isort]
 profile = black

From 921b2d08882636584dcad4fdb88d9413bd8240a0 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:34:52 +1000
Subject: [PATCH 08/66] Fix indents

---
 README.md | 216 +++++++++++++++++++++++++++---------------------------
 1 file changed, 108 insertions(+), 108 deletions(-)

diff --git a/README.md b/README.md
index 35159c2..dd65753 100644
--- a/README.md
+++ b/README.md
@@ -116,160 +116,160 @@ In case when the same option is specified both in the YAML file and in as a comm
 The following options are common between both the `archive` and `download` commands of the BDFR.
 
 - `directory`
-  - This is the directory to which the BDFR will download and place all files
+    - This is the directory to which the BDFR will download and place all files
 - `--authenticate`
-  - This flag will make the BDFR attempt to use an authenticated Reddit session
-  - See [Authentication](#authentication-and-security) for more details
+    - This flag will make the BDFR attempt to use an authenticated Reddit session
+    - See [Authentication](#authentication-and-security) for more details
 - `--config`
-  - If the path to a configuration file is supplied with this option, the BDFR will use the specified config
-  - See [Configuration Files](#configuration) for more details
+    - If the path to a configuration file is supplied with this option, the BDFR will use the specified config
+    - See [Configuration Files](#configuration) for more details
 - `--opts`
-  - Load options from a YAML file.
-  - Has higher prority than the global config file but lower than command-line arguments.
-  - See [opts_example.yaml](./opts_example.yaml) for an example file.
+    - Load options from a YAML file.
+    - Has higher prority than the global config file but lower than command-line arguments.
+    - See [opts_example.yaml](./opts_example.yaml) for an example file.
 - `--disable-module`
-  - Can be specified multiple times
-  - Disables certain modules from being used
-  - See [Disabling Modules](#disabling-modules) for more information and a list of module names
+    - Can be specified multiple times
+    - Disables certain modules from being used
+    - See [Disabling Modules](#disabling-modules) for more information and a list of module names
 - `--ignore-user`
-  - This will add a user to ignore
-  - Can be specified multiple times
+    - This will add a user to ignore
+    - Can be specified multiple times
 - `--include-id-file`
-  - This will add any submission with the IDs in the files provided
-  - Can be specified multiple times
-  - Format is one ID per line
+    - This will add any submission with the IDs in the files provided
+    - Can be specified multiple times
+    - Format is one ID per line
 - `--log`
-  - This allows one to specify the location of the logfile
-  - This must be done when running multiple instances of the BDFR, see [Multiple Instances](#multiple-instances) below
+    - This allows one to specify the location of the logfile
+    - This must be done when running multiple instances of the BDFR, see [Multiple Instances](#multiple-instances) below
 - `--saved`
-  - This option will make the BDFR use the supplied user's saved posts list as a download source
-  - This requires an authenticated Reddit instance, using the `--authenticate` flag, as well as `--user` set to `me`
+    - This option will make the BDFR use the supplied user's saved posts list as a download source
+    - This requires an authenticated Reddit instance, using the `--authenticate` flag, as well as `--user` set to `me`
 - `--search`
-  - This will apply the input search term to specific lists when scraping submissions
-  - A search term can only be applied when using the `--subreddit` and `--multireddit` flags
+    - This will apply the input search term to specific lists when scraping submissions
+    - A search term can only be applied when using the `--subreddit` and `--multireddit` flags
 - `--submitted`
-  - This will use a user's submissions as a source
-  - A user must be specified with `--user`
+    - This will use a user's submissions as a source
+    - A user must be specified with `--user`
 - `--upvoted`
-  - This will use a user's upvoted posts as a source of posts to scrape
-  - This requires an authenticated Reddit instance, using the `--authenticate` flag, as well as `--user` set to `me`
+    - This will use a user's upvoted posts as a source of posts to scrape
+    - This requires an authenticated Reddit instance, using the `--authenticate` flag, as well as `--user` set to `me`
 - `-L, --limit`
-  - This is the limit on the number of submissions retrieve
-  - Default is max possible
-  - Note that this limit applies to **each source individually** e.g. if a `--limit` of 10 and three subreddits are provided, then 30 total submissions will be scraped
-  - If it is not supplied, then the BDFR will default to the maximum allowed by Reddit, roughly 1000 posts. **We cannot bypass this.**
+    - This is the limit on the number of submissions retrieve
+    - Default is max possible
+    - Note that this limit applies to **each source individually** e.g. if a `--limit` of 10 and three subreddits are provided, then 30 total submissions will be scraped
+    - If it is not supplied, then the BDFR will default to the maximum allowed by Reddit, roughly 1000 posts. **We cannot bypass this.**
 - `-S, --sort`
-  - This is the sort type for each applicable submission source supplied to the BDFR
-  - This option does not apply to upvoted or saved posts when scraping from these sources
-  - The following options are available:
-    - `controversial`
-    - `hot` (default)
-    - `new`
-    - `relevance` (only available when using `--search`)
-    - `rising`
-    - `top`
+    - This is the sort type for each applicable submission source supplied to the BDFR
+    - This option does not apply to upvoted or saved posts when scraping from these sources
+    - The following options are available:
+        - `controversial`
+        - `hot` (default)
+        - `new`
+        - `relevance` (only available when using `--search`)
+        - `rising`
+        - `top`
 - `-l, --link`
-  - This is a direct link to a submission to download, either as a URL or an ID
-  - Can be specified multiple times
+    - This is a direct link to a submission to download, either as a URL or an ID
+    - Can be specified multiple times
 - `-m, --multireddit`
-  - This is the name of a multireddit to add as a source
-  - Can be specified multiple times
-    - This can be done by using `-m` multiple times
-    - Multireddits can also be used to provide CSV multireddits e.g. `-m 'chess, favourites'`
-  - The specified multireddits must all belong to the user specified with the `--user` option
+    - This is the name of a multireddit to add as a source
+    - Can be specified multiple times
+        - This can be done by using `-m` multiple times
+        - Multireddits can also be used to provide CSV multireddits e.g. `-m 'chess, favourites'`
+    - The specified multireddits must all belong to the user specified with the `--user` option
 - `-s, --subreddit`
-  - This adds a subreddit as a source
-  - Can be used mutliple times
-    - This can be done by using `-s` multiple times
-    - Subreddits can also be used to provide CSV subreddits e.g. `-m 'all, python, mindustry'`
+    - This adds a subreddit as a source
+    - Can be used mutliple times
+        - This can be done by using `-s` multiple times
+        - Subreddits can also be used to provide CSV subreddits e.g. `-m 'all, python, mindustry'`
 - `-t, --time`
-  - This is the time filter that will be applied to all applicable sources
-  - This option does not apply to upvoted or saved posts when scraping from these sources
-  - The following options are available:
-    - `all` (default)
-    - `hour`
-    - `day`
-    - `week`
-    - `month`
-    - `year`
-  - `--time-format`
-    - This specifies the format of the datetime string that replaces `{DATE}` in file and folder naming schemes
-    - See [Time Formatting Customisation](#time-formatting-customisation) for more details, and the formatting scheme
+    - This is the time filter that will be applied to all applicable sources
+    - This option does not apply to upvoted or saved posts when scraping from these sources
+    - The following options are available:
+        - `all` (default)
+        - `hour`
+        - `day`
+        - `week`
+        - `month`
+        - `year`
+    - `--time-format`
+        - This specifies the format of the datetime string that replaces `{DATE}` in file and folder naming schemes
+        - See [Time Formatting Customisation](#time-formatting-customisation) for more details, and the formatting scheme
 - `-u, --user`
-  - This specifies the user to scrape in concert with other options
-  - When using `--authenticate`, `--user me` can be used to refer to the authenticated user
-  - Can be specified multiple times for multiple users
-    - If downloading a multireddit, only one user can be specified
+    - This specifies the user to scrape in concert with other options
+    - When using `--authenticate`, `--user me` can be used to refer to the authenticated user
+    - Can be specified multiple times for multiple users
+        - If downloading a multireddit, only one user can be specified
 - `-v, --verbose`
-  - Increases the verbosity of the program
-  - Can be specified multiple times
+    - Increases the verbosity of the program
+    - Can be specified multiple times
 
 ### Downloader Options
 
 The following options apply only to the `download` command. This command downloads the files and resources linked to in the submission, or a text submission itself, to the disk in the specified directory.
 
 - `--make-hard-links`
-  - This flag will create hard links to an existing file when a duplicate is downloaded
-  - This will make the file appear in multiple directories while only taking the space of a single instance
+    - This flag will create hard links to an existing file when a duplicate is downloaded
+    - This will make the file appear in multiple directories while only taking the space of a single instance
 - `--max-wait-time`
-  - This option specifies the maximum wait time for downloading a resource
-  - The default is 120 seconds
-  - See [Rate Limiting](#rate-limiting) for details
+    - This option specifies the maximum wait time for downloading a resource
+    - The default is 120 seconds
+    - See [Rate Limiting](#rate-limiting) for details
 - `--no-dupes`
-  - This flag will not redownload files if they already exist somewhere in the root folder tree
-  - This is calculated by MD5 hash
+    - This flag will not redownload files if they already exist somewhere in the root folder tree
+    - This is calculated by MD5 hash
 - `--search-existing`
-  - This will make the BDFR compile the hashes for every file in `directory` and store them to remove duplicates if `--no-dupes` is also supplied
+    - This will make the BDFR compile the hashes for every file in `directory` and store them to remove duplicates if `--no-dupes` is also supplied
 - `--file-scheme`
-  - Sets the scheme for files
-  - Default is `{REDDITOR}_{TITLE}_{POSTID}`
-  - See [Folder and File Name Schemes](#folder-and-file-name-schemes) for more details
+    - Sets the scheme for files
+    - Default is `{REDDITOR}_{TITLE}_{POSTID}`
+    - See [Folder and File Name Schemes](#folder-and-file-name-schemes) for more details
 - `--folder-scheme`
-  - Sets the scheme for folders
-  - Default is `{SUBREDDIT}`
-  - See [Folder and File Name Schemes](#folder-and-file-name-schemes) for more details
+    - Sets the scheme for folders
+    - Default is `{SUBREDDIT}`
+    - See [Folder and File Name Schemes](#folder-and-file-name-schemes) for more details
 - `--exclude-id`
-  - This will skip the download of any submission with the ID provided
-  - Can be specified multiple times
+    - This will skip the download of any submission with the ID provided
+    - Can be specified multiple times
 - `--exclude-id-file`
-  - This will skip the download of any submission with any of the IDs in the files provided
-  - Can be specified multiple times
-  - Format is one ID per line
+    - This will skip the download of any submission with any of the IDs in the files provided
+    - Can be specified multiple times
+    - Format is one ID per line
 - `--skip-domain`
-  - This adds domains to the download filter i.e. submissions coming from these domains will not be downloaded
-  - Can be specified multiple times
-  - Domains must be supplied in the form `example.com` or `img.example.com`
+    - This adds domains to the download filter i.e. submissions coming from these domains will not be downloaded
+    - Can be specified multiple times
+    - Domains must be supplied in the form `example.com` or `img.example.com`
 - `--skip`
-  - This adds file types to the download filter i.e. submissions with one of the supplied file extensions will not be downloaded
-  - Can be specified multiple times
+    - This adds file types to the download filter i.e. submissions with one of the supplied file extensions will not be downloaded
+    - Can be specified multiple times
 - `--skip-subreddit`
-  - This skips all submissions from the specified subreddit
-  - Can be specified multiple times
-  - Also accepts CSV subreddit names
+    - This skips all submissions from the specified subreddit
+    - Can be specified multiple times
+    - Also accepts CSV subreddit names
 - `--min-score`
-  - This skips all submissions which have fewer than specified upvotes
+    - This skips all submissions which have fewer than specified upvotes
 - `--max-score`
-  - This skips all submissions which have more than specified upvotes
+    - This skips all submissions which have more than specified upvotes
 - `--min-score-ratio`
-  - This skips all submissions which have lower than specified upvote ratio
+    - This skips all submissions which have lower than specified upvote ratio
 - `--max-score-ratio`
-  - This skips all submissions which have higher than specified upvote ratio
+    - This skips all submissions which have higher than specified upvote ratio
 
 ### Archiver Options
 
 The following options are for the `archive` command specifically.
 
 - `--all-comments`
-  - When combined with the `--user` option, this will download all the user's comments
+    - When combined with the `--user` option, this will download all the user's comments
 - `-f, --format`
-  - This specifies the format of the data file saved to disk
-  - The following formats are available:
-    - `json` (default)
-    - `xml`
-    - `yaml`
+    - This specifies the format of the data file saved to disk
+    - The following formats are available:
+        - `json` (default)
+        - `xml`
+        - `yaml`
 - `--comment-context`
-  - This option will, instead of downloading an individual comment, download the submission that comment is a part of
-  - May result in a longer run time as it retrieves much more data
+    - This option will, instead of downloading an individual comment, download the submission that comment is a part of
+    - May result in a longer run time as it retrieves much more data
 
 ### Cloner Options
 
@@ -426,7 +426,7 @@ The logfiles that the BDFR outputs are consistent and quite detailed and in a fo
 - Redgifs
 - Vidble
 - YouTube
-  - Any source supported by [YT-DLP](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) should be compatable
+    - Any source supported by [YT-DLP](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md) should be compatable
 
 ## Contributing
 

From bfd2d31b7b0e94094d389207f7611a58c20ab205 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:46:32 +1000
Subject: [PATCH 09/66] Update markdown style

---
 .markdown_style.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.markdown_style.rb b/.markdown_style.rb
index 61f127b..32ee0b1 100644
--- a/.markdown_style.rb
+++ b/.markdown_style.rb
@@ -1,3 +1,4 @@
 all
 exclude_tag :line_length
 rule 'MD007', :indent => 4
+rule 'MD029', :style => 'ordered'

From 8feb6517f1be0d303792f1639845f11267f3d22e Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:46:45 +1000
Subject: [PATCH 10/66] Format supporting documents correctly

---
 docs/ARCHITECTURE.md | 36 ++++++++++++++----------------------
 docs/CONTRIBUTING.md | 24 ++++++++++++------------
 2 files changed, 26 insertions(+), 34 deletions(-)

diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 33d4297..8fc4e13 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -6,11 +6,11 @@ When the project was rewritten for v2, the goal was to make the codebase easily
 
 The BDFR is designed to be a stateless downloader. This means that the state of the program is forgotten between each run of the program. There are no central lists, databases, or indices, that the BDFR uses, only the actual files on disk. There are several advantages to this approach:
 
-  1. There is no chance of the database being corrupted or changed by something other than the BDFR, rendering the BDFR's "idea" of the archive wrong or incomplete.
-  2. Any information about the archive is contained by the archive itself i.e. for a list of all submission IDs in the archive, this can be extracted from the names of the files in said archive, assuming an appropriate naming scheme was used.
-  3. Archives can be merged, split, or editing without worrying about having to update a central database
-  4. There are no versioning issues between updates of the BDFR, where old version are stuck with a worse form of the database
-  5. An archive can be put on a USB, moved to another computer with possibly a very different BDFR version, and work completely fine
+1. There is no chance of the database being corrupted or changed by something other than the BDFR, rendering the BDFR's "idea" of the archive wrong or incomplete.
+2. Any information about the archive is contained by the archive itself i.e. for a list of all submission IDs in the archive, this can be extracted from the names of the files in said archive, assuming an appropriate naming scheme was used.
+3. Archives can be merged, split, or editing without worrying about having to update a central database
+4. There are no versioning issues between updates of the BDFR, where old version are stuck with a worse form of the database
+5. An archive can be put on a USB, moved to another computer with possibly a very different BDFR version, and work completely fine
 
 Another major part of the ethos of the design is DOTADIW, Do One Thing And Do It Well. It's a major part of Unix philosophy and states that each tool should have a well-defined, limited purpose. To this end, the BDFR is, as the name implies, a *downloader*. That is the scope of the tool. Managing the files downloaded can be for better-suited programs, since the BDFR is not a file manager. Nor the BDFR concern itself with how any of the data downloaded is displayed, changed, parsed, or analysed. This makes the BDFR suitable for data science-related tasks, archiving, personal downloads, or analysis of various Reddit sources as the BDFR is completely agnostic on how the data is used.
 
@@ -18,23 +18,15 @@ Another major part of the ethos of the design is DOTADIW, Do One Thing And Do It
 
 The BDFR is organised around a central object, the RedditDownloader class. The Archiver object extends and inherits from this class.
 
-  1. The RedditDownloader parses all the arguments and configuration options, held in the Configuration object, and creates a variety of internal objects for use, such as the file name formatter, download filter, etc.
-
-  2. The RedditDownloader scrapes raw submissions from Reddit via several methods relating to different sources. A source is defined as a single stream of submissions from a subreddit, multireddit, or user list.
-
-  3. These raw submissions are passed to the DownloaderFactory class to select the specialised downloader class to use. Each of these are for a specific website or link type, with some catch-all classes like Direct.
-
-  4. The BaseDownloader child, spawned by DownloaderFactory, takes the link and does any necessary processing to find the direct link to the actual resource.
-
-  5. This is returned to the RedditDownloader in the form of a Resource object. This holds the URL and some other information for the final resource.
-
-  6. The Resource is passed through the DownloadFilter instantiated in step 1.
-
-  7. The destination file name for the Resource is calculated. If it already exists, then the Resource will be discarded.
-
-  8. Here the actual data is downloaded to the Resource and a hash calculated which is used to find duplicates.
-
-  9. Only then is the Resource written to the disk.
+1. The RedditDownloader parses all the arguments and configuration options, held in the Configuration object, and creates a variety of internal objects for use, such as the file name formatter, download filter, etc.
+2. The RedditDownloader scrapes raw submissions from Reddit via several methods relating to different sources. A source is defined as a single stream of submissions from a subreddit, multireddit, or user list.
+3. These raw submissions are passed to the DownloaderFactory class to select the specialised downloader class to use. Each of these are for a specific website or link type, with some catch-all classes like Direct.
+4. The BaseDownloader child, spawned by DownloaderFactory, takes the link and does any necessary processing to find the direct link to the actual resource.
+5. This is returned to the RedditDownloader in the form of a Resource object. This holds the URL and some other information for the final resource.
+6. The Resource is passed through the DownloadFilter instantiated in step 1.
+7. The destination file name for the Resource is calculated. If it already exists, then the Resource will be discarded.
+8. Here the actual data is downloaded to the Resource and a hash calculated which is used to find duplicates.
+9. Only then is the Resource written to the disk.
 
 This is the step-by-step process that the BDFR goes through to download a Reddit post.
 
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 43d26f7..96e5e19 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -26,13 +26,13 @@ Before creating a pull request (PR), check out [ARCHITECTURE](ARCHITECTURE.md) f
 
 Once you have done both of these, the below list shows the path that should be followed when writing a PR.
 
-  1. If an issue does not already exist, open one that will relate to the PR.
-  2. Ensure that any changes fit into the architecture specified above.
-  3. Ensure that you have written tests that cover the new code.
-  4. Ensure that no existing tests fail, unless there is a good reason for them to do so.
-  5. If needed, update any documentation with changes.
-  6. Open a pull request that references the relevant issue.
-  7. Expect changes or suggestions and heed the Code of Conduct. We're all volunteers here.
+1. If an issue does not already exist, open one that will relate to the PR.
+2. Ensure that any changes fit into the architecture specified above.
+3. Ensure that you have written tests that cover the new code.
+4. Ensure that no existing tests fail, unless there is a good reason for them to do so.
+5. If needed, update any documentation with changes.
+6. Open a pull request that references the relevant issue.
+7. Expect changes or suggestions and heed the Code of Conduct. We're all volunteers here.
 
 Someone will review your pull request as soon as possible, but remember that all maintainers are volunteers and this won't happen immediately. Once it is approved, congratulations! Your code is now part of the BDFR.
 
@@ -87,14 +87,14 @@ When submitting a PR, it is required that you run **all** possible tests to ensu
 This is accomplished with marks, a system that pytest uses to categorise tests. There are currently the current marks in use in the BDFR test suite.
 
 - `slow`
-  - This marks a test that may take a long time to complete
-  - Usually marks a test that downloads many submissions or downloads a particularly large resource
+    - This marks a test that may take a long time to complete
+    - Usually marks a test that downloads many submissions or downloads a particularly large resource
 - `online`
-  - This marks a test that requires an internet connection and uses online resources
+    - This marks a test that requires an internet connection and uses online resources
 - `reddit`
-  - This marks a test that accesses online Reddit specifically
+    - This marks a test that accesses online Reddit specifically
 - `authenticated`
-  - This marks a test that requires a test configuration file with a valid OAuth2 token
+    - This marks a test that requires a test configuration file with a valid OAuth2 token
 
 These tests can be run either all at once, or excluding certain marks. The tests that require online resources, such as those marked `reddit` or `online`, will naturally require more time to run than tests that are entirely offline. To run tests, you must be in the root directory of the project and can use the following command.
 

From 47e49a2e98985f6aeef6a7971557dca9bcaa66a5 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:48:11 +1000
Subject: [PATCH 11/66] Reorder workflow dependencies

---
 .github/workflows/formatting_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index 4941d83..39848c8 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: paolorechia/pox@v1.0.1
       - uses: actionshub/markdownlint@main
+      - uses: paolorechia/pox@v1.0.1
         with:
           tox_env: "format_check"
\ No newline at end of file

From 8cfc3140380dc9f29c215228f4567b9e6b451d68 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:51:18 +1000
Subject: [PATCH 12/66] Install mdl in workflow

---
 .github/workflows/formatting_check.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index 39848c8..79ccb4b 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -5,8 +5,9 @@ jobs:
   formatting_check:
     runs-on: ubuntu-latest
     steps:
+      - name: Install dependencies
+        run: gem install mdl
       - uses: actions/checkout@v3
-      - uses: actionshub/markdownlint@main
       - uses: paolorechia/pox@v1.0.1
         with:
           tox_env: "format_check"
\ No newline at end of file

From 614c19be109effff93e47ae26f0ba9bf5503a9e6 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 3 Dec 2022 16:56:44 +1000
Subject: [PATCH 13/66] Fix workflow error

---
 .github/workflows/formatting_check.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index 79ccb4b..deb44d5 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Install dependencies
-        run: gem install mdl
+        run: sudo gem install mdl
       - uses: actions/checkout@v3
       - uses: paolorechia/pox@v1.0.1
         with:

From d4bfe8fa194b7dac3b8c71a6fb7af1644d45bc95 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 3 Dec 2022 14:49:39 -0500
Subject: [PATCH 14/66] Formatting cleanup

Cleanup some formatting from switch to Black
---
 bdfr/archiver.py         | 2 +-
 tests/test_downloader.py | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/bdfr/archiver.py b/bdfr/archiver.py
index 3d0d31b..28a270b 100644
--- a/bdfr/archiver.py
+++ b/bdfr/archiver.py
@@ -35,7 +35,7 @@ class Archiver(RedditConnector):
                     ):
                         logger.debug(
                             f"Submission {submission.id} in {submission.subreddit.display_name} skipped"
-                            f' due to {submission.author.name if submission.author else "DELETED"} being an ignored user'
+                            f" due to {submission.author.name if submission.author else 'DELETED'} being an ignored user"
                         )
                         continue
                     if submission.id in self.excluded_submission_ids:
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index 7b81a85..ba81b80 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -167,9 +167,7 @@ def test_download_submission_file_exists(
     folder_contents = list(tmp_path.iterdir())
     output = capsys.readouterr()
     assert len(folder_contents) == 1
-    assert (
-        "Arneeman_Metagaming isn't always a bad thing_m1hqw6.png" " from submission m1hqw6 already exists" in output.out
-    )
+    assert "Arneeman_Metagaming isn't always a bad thing_m1hqw6.png from submission m1hqw6 already exists" in output.out
 
 
 @pytest.mark.online

From 8af00b20bcb9ce90479c6f93569ae53b892a53de Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sun, 4 Dec 2022 14:49:37 +1000
Subject: [PATCH 15/66] Move formatter settings

---
 pyproject.toml | 7 +++++++
 tox.ini        | 9 ++-------
 2 files changed, 9 insertions(+), 7 deletions(-)
 create mode 100644 pyproject.toml

diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..4dced2f
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,7 @@
+[tool.black]
+line-length = 120
+
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 120
diff --git a/tox.ini b/tox.ini
index 88df732..b50927c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,7 +10,7 @@ deps =
 skip_install = True
 commands =
     isort bdfr tests
-    black bdfr tests --line-length 120
+    black bdfr tests
 
 [testenv:format_check]
 deps =
@@ -20,10 +20,5 @@ skip_install = True
 allowlist_externals = mdl
 commands =
     isort bdfr tests --check
-    black bdfr tests --line-length 120 --check
+    black bdfr tests --check
     mdl README.md docs/ -s .markdown_style.rb
-
-[isort]
-profile = black
-multi_line_output = 3
-line_length = 120

From 7e3b11caf851cb524b80504395816f6f5f7b1637 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sun, 4 Dec 2022 14:58:54 +1000
Subject: [PATCH 16/66] Add support for pre-commit

---
 .pre-commit-config.yaml | 19 +++++++++++++++++++
 dev-requirements.txt    |  1 +
 2 files changed, 20 insertions(+)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..7013228
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,19 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        name: isort (python)
+
+  - repo: https://github.com/markdownlint/markdownlint
+    rev: v0.12.0
+    hooks:
+      - id: markdownlint
diff --git a/dev-requirements.txt b/dev-requirements.txt
index af48d1b..34597e7 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,4 +1,5 @@
 black
 isort
+pre-commit
 pytest
 tox

From 628739d0b8320f5a76c5badb302fb90dc23732d0 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Thu, 8 Dec 2022 15:46:53 +1000
Subject: [PATCH 17/66] Add markdownlint default file

---
 .mdlrc  | 1 +
 tox.ini | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 .mdlrc

diff --git a/.mdlrc b/.mdlrc
new file mode 100644
index 0000000..dd06c8c
--- /dev/null
+++ b/.mdlrc
@@ -0,0 +1 @@
+style "#{File.dirname(__FILE__)}/.markdown_style.rb"
diff --git a/tox.ini b/tox.ini
index b50927c..be451e6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -21,4 +21,4 @@ allowlist_externals = mdl
 commands =
     isort bdfr tests --check
     black bdfr tests --check
-    mdl README.md docs/ -s .markdown_style.rb
+    mdl README.md docs/

From 1bc20f238e1c0eba396d1ed3d2e329f2454c0ced Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Thu, 8 Dec 2022 15:46:58 +1000
Subject: [PATCH 18/66] Update CONTRIBUTING to include new tools

---
 docs/CONTRIBUTING.md | 41 +++++++++++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 96e5e19..ea2d37f 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -58,23 +58,36 @@ Then, you can run the program from anywhere in your disk as such:
 bdfr
 ```
 
-## Style Guide
+There are additional Python packages that are required to develop the BDFR. These can be installed with the following command:
 
-The BDFR must conform to PEP8 standard wherever there is Python code, with one exception. Line lengths may extend to 120 characters, but all other PEP8 standards must be followed.
-
-It's easy to format your code without any manual work via a variety of tools. Autopep8 is a good one, and can be used with `autopep8 --max-line-length 120` which will format the code according to the style in use with the BDFR.
-
-Hanging brackets are preferred when there are many items, items that otherwise go over the 120 character line limit, or when doing so would increase readability. It is also preferred when there might be many commits altering the list, such as with the parameter lists for tests. A hanging comma is also required in such cases. An example of this is below:
-
-```python
-test = [
-    'test 1',
-    'test 2',
-    'test 3',
-]
+```bash
+python3 -m pip install -r dev-requirements.txt
 ```
 
-Note that the last bracket is on its own line, and that the first bracket has a new line before the first term. Also note that there is a comma after the last term.
+### Tools
+
+The BDFR project uses several tools to manage the code of the project. These include:
+
+- [black](https://github.com/psf/black)
+- [isort](https://github.com/PyCQA/isort)
+- [markdownlint (mdl)](https://github.com/markdownlint/markdownlint)
+- [tox](https://tox.wiki/en/latest/)
+- [pre-commit](https://github.com/pre-commit/pre-commit)
+
+The first three tools are formatters. These change the code to the standards expected for the BDFR project. The configuration details for these tools are contained in the [pyproject.toml](../pyproject.toml) file for the project.
+
+The tool `tox` is used to run tests and tools on demand and has the following environments:
+
+- `format`
+- `format_check`
+
+The tool `pre-commit` is optional, and runs the three formatting tools automatically when a commit is made. This is **highly recommended** to ensure that all code submitted for this project is formatted acceptably. Note that any PR that does not follow the formatting guide will not be accepted. For information on how to use pre-commit to avoid this, see [the pre-commit documentation](https://pre-commit.com/).
+
+## Style Guide
+
+The BDFR uses the Black formatting standard and enforces this with the tool by the same name. Additionally, the tool isort is used as well to format imports.
+
+See [Preparing the Environment for Development](#preparing-the-environment-for-development) for how to setup these tools to run automatically.
 
 ## Tests
 

From 3aa740e979ffd976508009a8e4c80934eeb3eaeb Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 10 Dec 2022 12:36:54 -0500
Subject: [PATCH 19/66] Add soft fail on 5xx Prawcore errors.

---
 bdfr/connector.py  | 44 +++++++++++++++++++++++++-------------------
 bdfr/downloader.py | 16 +++++++++++-----
 2 files changed, 36 insertions(+), 24 deletions(-)

diff --git a/bdfr/connector.py b/bdfr/connector.py
index ea970db..e5d74a2 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -13,6 +13,7 @@ from abc import ABCMeta, abstractmethod
 from datetime import datetime
 from enum import Enum, auto
 from pathlib import Path
+from time import sleep
 from typing import Callable, Iterator
 
 import appdirs
@@ -353,26 +354,31 @@ class RedditConnector(metaclass=ABCMeta):
             generators = []
             for user in self.args.user:
                 try:
-                    self.check_user_existence(user)
-                except errors.BulkDownloaderException as e:
-                    logger.error(e)
-                    continue
-                if self.args.submitted:
-                    logger.debug(f"Retrieving submitted posts of user {self.args.user}")
-                    generators.append(
-                        self.create_filtered_listing_generator(
-                            self.reddit_instance.redditor(user).submissions,
+                    try:
+                        self.check_user_existence(user)
+                    except errors.BulkDownloaderException as e:
+                        logger.error(e)
+                        continue
+                    if self.args.submitted:
+                        logger.debug(f"Retrieving submitted posts of user {user}")
+                        generators.append(
+                            self.create_filtered_listing_generator(
+                                self.reddit_instance.redditor(user).submissions,
+                            )
                         )
-                    )
-                if not self.authenticated and any((self.args.upvoted, self.args.saved)):
-                    logger.warning("Accessing user lists requires authentication")
-                else:
-                    if self.args.upvoted:
-                        logger.debug(f"Retrieving upvoted posts of user {self.args.user}")
-                        generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit))
-                    if self.args.saved:
-                        logger.debug(f"Retrieving saved posts of user {self.args.user}")
-                        generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit))
+                    if not self.authenticated and any((self.args.upvoted, self.args.saved)):
+                        logger.warning("Accessing user lists requires authentication")
+                    else:
+                        if self.args.upvoted:
+                            logger.debug(f"Retrieving upvoted posts of user {user}")
+                            generators.append(self.reddit_instance.redditor(user).upvoted(limit=self.args.limit))
+                        if self.args.saved:
+                            logger.debug(f"Retrieving saved posts of user {user}")
+                            generators.append(self.reddit_instance.redditor(user).saved(limit=self.args.limit))
+                except prawcore.PrawcoreException as e:
+                    logger.error(f"User {user} failed to be retrieved due to a PRAW exception: {e}")
+                    logger.debug("Waiting 60 seconds to continue")
+                    sleep(60)
             return generators
         else:
             return []
diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index fa5d10c..1cb6d46 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -8,6 +8,7 @@ import time
 from datetime import datetime
 from multiprocessing import Pool
 from pathlib import Path
+from time import sleep
 
 import praw
 import praw.exceptions
@@ -42,11 +43,16 @@ class RedditDownloader(RedditConnector):
 
     def download(self):
         for generator in self.reddit_lists:
-            for submission in generator:
-                try:
-                    self._download_submission(submission)
-                except prawcore.PrawcoreException as e:
-                    logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
+            try:
+                for submission in generator:
+                    try:
+                        self._download_submission(submission)
+                    except prawcore.PrawcoreException as e:
+                        logger.error(f"Submission {submission.id} failed to download due to a PRAW exception: {e}")
+            except prawcore.PrawcoreException as e:
+                logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
+                logger.debug("Waiting 60 seconds to continue")
+                sleep(60)
 
     def _download_submission(self, submission: praw.models.Submission):
         if submission.id in self.excluded_submission_ids:

From ac91c9089c652e77a23ea44a556632e4d8e17636 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 10 Dec 2022 21:19:29 -0500
Subject: [PATCH 20/66] Add 5xx soft fail for clone/archive

---
 bdfr/archiver.py | 40 +++++++++++++++++++++++-----------------
 bdfr/cloner.py   | 18 ++++++++++++------
 2 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/bdfr/archiver.py b/bdfr/archiver.py
index 28a270b..e2ed33d 100644
--- a/bdfr/archiver.py
+++ b/bdfr/archiver.py
@@ -4,6 +4,7 @@
 import json
 import logging
 import re
+from time import sleep
 from typing import Iterator, Union
 
 import dict2xml
@@ -28,23 +29,28 @@ class Archiver(RedditConnector):
 
     def download(self):
         for generator in self.reddit_lists:
-            for submission in generator:
-                try:
-                    if (submission.author and submission.author.name in self.args.ignore_user) or (
-                        submission.author is None and "DELETED" in self.args.ignore_user
-                    ):
-                        logger.debug(
-                            f"Submission {submission.id} in {submission.subreddit.display_name} skipped"
-                            f" due to {submission.author.name if submission.author else 'DELETED'} being an ignored user"
-                        )
-                        continue
-                    if submission.id in self.excluded_submission_ids:
-                        logger.debug(f"Object {submission.id} in exclusion list, skipping")
-                        continue
-                    logger.debug(f"Attempting to archive submission {submission.id}")
-                    self.write_entry(submission)
-                except prawcore.PrawcoreException as e:
-                    logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
+            try:
+                for submission in generator:
+                    try:
+                        if (submission.author and submission.author.name in self.args.ignore_user) or (
+                            submission.author is None and "DELETED" in self.args.ignore_user
+                        ):
+                            logger.debug(
+                                f"Submission {submission.id} in {submission.subreddit.display_name} skipped due to"
+                                f" {submission.author.name if submission.author else 'DELETED'} being an ignored user"
+                            )
+                            continue
+                        if submission.id in self.excluded_submission_ids:
+                            logger.debug(f"Object {submission.id} in exclusion list, skipping")
+                            continue
+                        logger.debug(f"Attempting to archive submission {submission.id}")
+                        self.write_entry(submission)
+                    except prawcore.PrawcoreException as e:
+                        logger.error(f"Submission {submission.id} failed to be archived due to a PRAW exception: {e}")
+            except prawcore.PrawcoreException as e:
+                logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
+                logger.debug("Waiting 60 seconds to continue")
+                sleep(60)
 
     def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
         supplied_submissions = []
diff --git a/bdfr/cloner.py b/bdfr/cloner.py
index c26d17b..e82cfaa 100644
--- a/bdfr/cloner.py
+++ b/bdfr/cloner.py
@@ -2,6 +2,7 @@
 # coding=utf-8
 
 import logging
+from time import sleep
 
 import prawcore
 
@@ -18,9 +19,14 @@ class RedditCloner(RedditDownloader, Archiver):
 
     def download(self):
         for generator in self.reddit_lists:
-            for submission in generator:
-                try:
-                    self._download_submission(submission)
-                    self.write_entry(submission)
-                except prawcore.PrawcoreException as e:
-                    logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
+            try:
+                for submission in generator:
+                    try:
+                        self._download_submission(submission)
+                        self.write_entry(submission)
+                    except prawcore.PrawcoreException as e:
+                        logger.error(f"Submission {submission.id} failed to be cloned due to a PRAW exception: {e}")
+            except prawcore.PrawcoreException as e:
+                logger.error(f"The submission after {submission.id} failed to download due to a PRAW exception: {e}")
+                logger.debug("Waiting 60 seconds to continue")
+                sleep(60)

From 15a9d25a9db82ab543b9775349a90d2a69c3f5a7 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Sun, 11 Dec 2022 14:20:04 -0500
Subject: [PATCH 21/66] Imgur webp coverage

update regex to catch _d in webp links from imgur.
---
 bdfr/site_downloaders/imgur.py       | 2 +-
 tests/site_downloaders/test_imgur.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index f91e34f..0b9ecdd 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -41,7 +41,7 @@ class Imgur(BaseDownloader):
     @staticmethod
     def _get_data(link: str) -> dict:
         try:
-            imgur_id = re.match(r".*/(.*?)(\..{0,})?$", link).group(1)
+            imgur_id = re.match(r".*/(.*?)(_d)?(\..{0,})?$", link).group(1)
             gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else ""
             link = f"https://imgur.com/{gallery}{imgur_id}"
         except AttributeError:
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 38dbdc5..6b49cd5 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -170,6 +170,10 @@ def test_imgur_extension_validation_bad(test_extension: str):
             "http://i.imgur.com/s9uXxlq.jpg?5.jpg",
             ("338de3c23ee21af056b3a7c154e2478f",),
         ),
+        (
+            "https://i.imgur.com/2TtN68l_d.webp",
+            ("6569ab9ad9fa68d93f6b408f112dd741",),
+        ),
     ),
 )
 def test_find_resources(test_url: str, expected_hashes: list[str]):

From 4ba5df6b3728c08980ab1fe3f99bbf051a8168c3 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Wed, 14 Dec 2022 23:04:33 -0500
Subject: [PATCH 22/66] 5xx error tests

---
 .../test_archive_integration.py               | 29 +++++++++++++++++++
 .../test_clone_integration.py                 | 29 +++++++++++++++++++
 .../test_download_integration.py              | 29 +++++++++++++++++++
 3 files changed, 87 insertions(+)

diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py
index f10f37c..1c0d30a 100644
--- a/tests/integration_tests/test_archive_integration.py
+++ b/tests/integration_tests/test_archive_integration.py
@@ -4,7 +4,9 @@
 import re
 import shutil
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
+import prawcore
 import pytest
 from click.testing import CliRunner
 
@@ -176,3 +178,30 @@ def test_cli_archive_soft_fail(test_args: list[str], tmp_path: Path):
     assert result.exit_code == 0
     assert "failed to be archived due to a PRAW exception" in result.output
     assert "Attempting to archive" not in result.output
+
+
+@pytest.mark.skipif(not does_test_config_exist, reason="A test config file is required for integration tests")
+@pytest.mark.parametrize(
+    ("test_args", "response"),
+    (
+        (
+            ["--user", "nasa", "--submitted"],
+            502,
+        ),
+        (
+            ["--user", "nasa", "--submitted"],
+            504,
+        ),
+    ),
+)
+def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
+    runner = CliRunner()
+    test_args = create_basic_args_for_archive_runner(test_args, tmp_path)
+    with patch("bdfr.connector.sleep", return_value=None):
+        with patch(
+            "bdfr.connector.RedditConnector.check_user_existence",
+            side_effect=prawcore.exceptions.ResponseException(MagicMock(status_code=response)),
+        ):
+            result = runner.invoke(cli, test_args)
+            assert result.exit_code == 0
+            assert f"received {response} HTTP response" in result.output
diff --git a/tests/integration_tests/test_clone_integration.py b/tests/integration_tests/test_clone_integration.py
index e8dc008..eb64364 100644
--- a/tests/integration_tests/test_clone_integration.py
+++ b/tests/integration_tests/test_clone_integration.py
@@ -3,7 +3,9 @@
 
 import shutil
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
+import prawcore
 import pytest
 from click.testing import CliRunner
 
@@ -68,3 +70,30 @@ def test_cli_scrape_soft_fail(test_args: list[str], tmp_path: Path):
     assert result.exit_code == 0
     assert "Downloaded submission" not in result.output
     assert "Record for entry item" not in result.output
+
+
+@pytest.mark.skipif(not does_test_config_exist, reason="A test config file is required for integration tests")
+@pytest.mark.parametrize(
+    ("test_args", "response"),
+    (
+        (
+            ["--user", "nasa", "--submitted"],
+            502,
+        ),
+        (
+            ["--user", "nasa", "--submitted"],
+            504,
+        ),
+    ),
+)
+def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
+    runner = CliRunner()
+    test_args = create_basic_args_for_cloner_runner(test_args, tmp_path)
+    with patch("bdfr.connector.sleep", return_value=None):
+        with patch(
+            "bdfr.connector.RedditConnector.check_user_existence",
+            side_effect=prawcore.exceptions.ResponseException(MagicMock(status_code=response)),
+        ):
+            result = runner.invoke(cli, test_args)
+            assert result.exit_code == 0
+            assert f"received {response} HTTP response" in result.output
diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 2ab38a0..e44c95e 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -3,7 +3,9 @@
 
 import shutil
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
+import prawcore
 import pytest
 from click.testing import CliRunner
 
@@ -396,3 +398,30 @@ def test_cli_download_score_filter(test_args: list[str], was_filtered: bool, tmp
     result = runner.invoke(cli, test_args)
     assert result.exit_code == 0
     assert ("filtered due to score" in result.output) == was_filtered
+
+
+@pytest.mark.skipif(not does_test_config_exist, reason="A test config file is required for integration tests")
+@pytest.mark.parametrize(
+    ("test_args", "response"),
+    (
+        (
+            ["--user", "nasa", "--submitted"],
+            502,
+        ),
+        (
+            ["--user", "nasa", "--submitted"],
+            504,
+        ),
+    ),
+)
+def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
+    runner = CliRunner()
+    test_args = create_basic_args_for_download_runner(test_args, tmp_path)
+    with patch("bdfr.connector.sleep", return_value=None):
+        with patch(
+            "bdfr.connector.RedditConnector.check_user_existence",
+            side_effect=prawcore.exceptions.ResponseException(MagicMock(status_code=response)),
+        ):
+            result = runner.invoke(cli, test_args)
+            assert result.exit_code == 0
+            assert f"received {response} HTTP response" in result.output

From e32d322dbd2d5532201bdf5123e0dbe71ec0179f Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Fri, 16 Dec 2022 14:56:44 -0500
Subject: [PATCH 23/66] Add shell completions

---
 README.md          | 10 +++++---
 bdfr/__main__.py   | 33 ++++++++++++++++++++++++++
 bdfr/completion.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 3 deletions(-)
 create mode 100644 bdfr/completion.py

diff --git a/README.md b/README.md
index dd65753..4b634ec 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,9 @@ Included in this README are a few example Bash tricks to get certain behaviour.
 
 ## Installation
 
-*Bulk Downloader for Reddit* needs Python version 3.9 or above. Please update Python before installation to meet the requirement. Then, you can install it via pip with:
+*Bulk Downloader for Reddit* needs Python version 3.9 or above. Please update Python before installation to meet the requirement.
+
+Then, you can install it via pip with:
 
 ```bash
 python3 -m pip install bdfr --upgrade
@@ -21,10 +23,12 @@ python3 -m pip install bdfr --upgrade
 or via [pipx](https://pypa.github.io/pipx) with:
 
 ```bash
-python3 -m pipx install bdfr --upgrade
+python3 -m pipx install bdfr
 ```
 
-**To update BDFR**, run the above command again after the installation.
+**To update BDFR**, run the above command again for pip or `pipx upgrade bdfr` for pipx installations.
+
+**To install shell completions**, run `bdfr completions`
 
 ### AUR Package
 
diff --git a/bdfr/__main__.py b/bdfr/__main__.py
index c26f577..d0c6664 100644
--- a/bdfr/__main__.py
+++ b/bdfr/__main__.py
@@ -7,6 +7,7 @@ import click
 
 from bdfr.archiver import Archiver
 from bdfr.cloner import RedditCloner
+from bdfr.completion import Completion
 from bdfr.configuration import Configuration
 from bdfr.downloader import RedditDownloader
 
@@ -74,15 +75,19 @@ def _add_options(opts: list):
 
 
 @click.group()
+@click.help_option("-h", "--help")
 def cli():
+    """BDFR is used to download and archive content from Reddit."""
     pass
 
 
 @cli.command("download")
 @_add_options(_common_options)
 @_add_options(_downloader_options)
+@click.help_option("-h", "--help")
 @click.pass_context
 def cli_download(context: click.Context, **_):
+    """Used to download content posted to Reddit."""
     config = Configuration()
     config.process_click_arguments(context)
     setup_logging(config.verbose)
@@ -99,8 +104,10 @@ def cli_download(context: click.Context, **_):
 @cli.command("archive")
 @_add_options(_common_options)
 @_add_options(_archiver_options)
+@click.help_option("-h", "--help")
 @click.pass_context
 def cli_archive(context: click.Context, **_):
+    """Used to archive post data from Reddit."""
     config = Configuration()
     config.process_click_arguments(context)
     setup_logging(config.verbose)
@@ -118,8 +125,10 @@ def cli_archive(context: click.Context, **_):
 @_add_options(_common_options)
 @_add_options(_archiver_options)
 @_add_options(_downloader_options)
+@click.help_option("-h", "--help")
 @click.pass_context
 def cli_clone(context: click.Context, **_):
+    """Combines archive and download commands."""
     config = Configuration()
     config.process_click_arguments(context)
     setup_logging(config.verbose)
@@ -133,6 +142,30 @@ def cli_clone(context: click.Context, **_):
         logger.info("Program complete")
 
 
+@cli.command("completion")
+@click.argument("shell", type=click.Choice(("all", "bash", "fish", "zsh"), case_sensitive=False), default="all")
+@click.help_option("-h", "--help")
+@click.option("-u", "--uninstall", is_flag=True, default=False, help="Uninstall completion")
+def cli_completion(shell: str, uninstall: bool):
+    """\b
+    Installs shell completions for BDFR.
+    Options: all, bash, fish, zsh
+    Default: all"""
+    shell = shell.lower()
+    if sys.platform == "win32":
+        print("Completions are not currently supported on Windows.")
+        return
+    if uninstall and click.confirm(f"Would you like to uninstall {shell} completions for BDFR"):
+        Completion(shell).uninstall()
+        return
+    if shell not in ("all", "bash", "fish", "zsh"):
+        print(f"{shell} is not a valid option.")
+        print("Options: all, bash, fish, zsh")
+        return
+    if click.confirm(f"Would you like to install {shell} completions for BDFR"):
+        Completion(shell).install()
+
+
 def setup_logging(verbosity: int):
     class StreamExceptionFilter(logging.Filter):
         def filter(self, record: logging.LogRecord) -> bool:
diff --git a/bdfr/completion.py b/bdfr/completion.py
new file mode 100644
index 0000000..1902319
--- /dev/null
+++ b/bdfr/completion.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+import os
+import subprocess
+
+import appdirs
+
+
+class Completion:
+    def __init__(self, shell: str):
+        self.shell = shell
+        self.env = os.environ.copy()
+        self.share_dir = appdirs.user_data_dir()
+        self.entry_points = ["bdfr"]
+
+    def install(self):
+        if self.shell in ("all", "bash"):
+            comp_dir = self.share_dir + "/bash-completion/completions/"
+            for point in self.entry_points:
+                self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "bash_source"
+                with open(comp_dir + point, "w") as file:
+                    file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
+                    print(f"Bash completion for {point} written to {comp_dir}{point}")
+        if self.shell in ("all", "fish"):
+            comp_dir = self.share_dir + "/fish/vendor_completions.d/"
+            for point in self.entry_points:
+                self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "fish_source"
+                with open(comp_dir + point, "w") as file:
+                    file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
+                    print(f"Fish completion for {point} written to {comp_dir}{point}")
+        if self.shell in ("all", "zsh"):
+            comp_dir = self.share_dir + "/zsh/site-functions/"
+            for point in self.entry_points:
+                self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "zsh_source"
+                with open(comp_dir + point, "w") as file:
+                    file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
+                    print(f"Zsh completion for {point} written to {comp_dir}{point}")
+
+    def uninstall(self):
+        if self.shell in ("all", "bash"):
+            comp_dir = self.share_dir + "/bash-completion/completions/"
+            for point in self.entry_points:
+                if os.path.exists(comp_dir + point):
+                    os.remove(comp_dir + point)
+                    print(f"Bash completion for {point} removed from {comp_dir}{point}")
+        if self.shell in ("all", "fish"):
+            comp_dir = self.share_dir + "/fish/vendor_completions.d/"
+            for point in self.entry_points:
+                if os.path.exists(comp_dir + point):
+                    os.remove(comp_dir + point)
+                    print(f"Fish completion for {point} removed from {comp_dir}{point}")
+        if self.shell in ("all", "zsh"):
+            comp_dir = self.share_dir + "/zsh/site-functions/"
+            for point in self.entry_points:
+                if os.path.exists(comp_dir + point):
+                    os.remove(comp_dir + point)
+                    print(f"Zsh completion for {point} removed from {comp_dir}{point}")

From 8c01a9e7a04bdbe759ee6f739a095b9b8329a462 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Fri, 16 Dec 2022 23:45:36 -0500
Subject: [PATCH 24/66] Consolidate to pyproject

Consolidates configs to pyproject.toml and updates workflows accordingly as well as sets sane minimums for dev requirements.

adds version check to main script.
---
 .github/workflows/formatting_check.yml |  2 +-
 .github/workflows/publish.yml          | 14 ++---
 .github/workflows/test.yml             | 10 ++--
 README.md                              |  2 +
 bdfr/__init__.py                       |  1 +
 bdfr/__main__.py                       | 19 +++++++
 bdfr/completion.py                     |  2 +-
 dev-requirements.txt                   |  5 --
 docs/CONTRIBUTING.md                   |  2 +-
 pyproject.toml                         | 75 ++++++++++++++++++++++++++
 pytest.ini                             |  7 ---
 requirements.txt                       |  9 ----
 setup.cfg                              | 26 ---------
 setup.py                               |  6 ---
 tox.ini                                |  2 +
 15 files changed, 114 insertions(+), 68 deletions(-)
 delete mode 100644 dev-requirements.txt
 delete mode 100644 pytest.ini
 delete mode 100644 requirements.txt
 delete mode 100644 setup.cfg
 delete mode 100644 setup.py

diff --git a/.github/workflows/formatting_check.yml b/.github/workflows/formatting_check.yml
index deb44d5..5a45479 100644
--- a/.github/workflows/formatting_check.yml
+++ b/.github/workflows/formatting_check.yml
@@ -10,4 +10,4 @@ jobs:
       - uses: actions/checkout@v3
       - uses: paolorechia/pox@v1.0.1
         with:
-          tox_env: "format_check"
\ No newline at end of file
+          tox_env: "format_check"
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 6f15a00..589c201 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -11,25 +11,25 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: '3.9'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install build setuptools wheel twine
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         twine upload dist/*
-    
-    - name: Upload coverage report
-      uses: actions/upload-artifact@v2
+
+    - name: Upload dist folder
+      uses: actions/upload-artifact@v3
       with:
         name: dist
         path: dist/
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5aa8c61..0d52ef6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -19,16 +19,16 @@ jobs:
           python-version: 3.9
           ext: .ps1
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Setup Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip flake8 pytest pytest-cov
-        pip install -r requirements.txt
+        pip install .
 
     - name: Make configuration for tests
       env:
@@ -43,9 +43,9 @@ jobs:
     - name: Test with pytest
       run: |
         pytest -m 'not slow' --verbose --cov=./bdfr/ --cov-report term:skip-covered --cov-report html
-        
+
     - name: Upload coverage report
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: coverage_report
         path: htmlcov/
diff --git a/README.md b/README.md
index 4b634ec..f732f6e 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,8 @@ python3 -m pipx install bdfr
 
 **To update BDFR**, run the above command again for pip or `pipx upgrade bdfr` for pipx installations.
 
+**To check your version of BDFR**, run `bdfr --version`
+
 **To install shell completions**, run `bdfr completions`
 
 ### AUR Package
diff --git a/bdfr/__init__.py b/bdfr/__init__.py
index e69de29..b482efe 100644
--- a/bdfr/__init__.py
+++ b/bdfr/__init__.py
@@ -0,0 +1 @@
+__version__ = "2.6.2"
diff --git a/bdfr/__main__.py b/bdfr/__main__.py
index d0c6664..57373c9 100644
--- a/bdfr/__main__.py
+++ b/bdfr/__main__.py
@@ -4,7 +4,9 @@ import logging
 import sys
 
 import click
+import requests
 
+from bdfr import __version__
 from bdfr.archiver import Archiver
 from bdfr.cloner import RedditCloner
 from bdfr.completion import Completion
@@ -74,8 +76,25 @@ def _add_options(opts: list):
     return wrap
 
 
+def _check_version(context, param, value):
+    if not value or context.resilient_parsing:
+        return
+    current = __version__
+    latest = requests.get("https://pypi.org/pypi/bdfr/json").json()["info"]["version"]
+    print(f"You are currently using v{current} the latest is v{latest}")
+    context.exit()
+
+
 @click.group()
 @click.help_option("-h", "--help")
+@click.option(
+    "--version",
+    is_flag=True,
+    is_eager=True,
+    expose_value=False,
+    callback=_check_version,
+    help="Check version and exit.",
+)
 def cli():
     """BDFR is used to download and archive content from Reddit."""
     pass
diff --git a/bdfr/completion.py b/bdfr/completion.py
index 1902319..8f4f122 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -12,7 +12,7 @@ class Completion:
         self.shell = shell
         self.env = os.environ.copy()
         self.share_dir = appdirs.user_data_dir()
-        self.entry_points = ["bdfr"]
+        self.entry_points = ["bdfr", "bdfr-archive", "bdfr-clone", "bdfr-download"]
 
     def install(self):
         if self.shell in ("all", "bash"):
diff --git a/dev-requirements.txt b/dev-requirements.txt
deleted file mode 100644
index 34597e7..0000000
--- a/dev-requirements.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-black
-isort
-pre-commit
-pytest
-tox
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index ea2d37f..72666e7 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -61,7 +61,7 @@ bdfr
 There are additional Python packages that are required to develop the BDFR. These can be installed with the following command:
 
 ```bash
-python3 -m pip install -r dev-requirements.txt
+python3 -m pip install -e .[dev]
 ```
 
 ### Tools
diff --git a/pyproject.toml b/pyproject.toml
index 4dced2f..23ae690 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,7 +1,82 @@
+[build-system]
+requires = ["setuptools>=65.6.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "bdfr"
+description = "Downloads and archives content from reddit"
+readme = "README.md"
+requires-python = ">=3.9"
+license = {file = "LICENSE"}
+keywords = ["reddit", "download", "archive",]
+authors = [{name = "Ali Parlakci", email = "parlakciali@gmail.com"}]
+maintainers = [{name = "Serene Arc", email = "serenical@gmail.com"}]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "appdirs>=1.4.4",
+    "beautifulsoup4>=4.10.0",
+    "click>=8.0.0",
+    "dict2xml>=1.7.0",
+    "praw>=7.2.0",
+    "pyyaml>=5.4.1",
+    "requests>=2.25.1",
+    "yt-dlp>=2022.11.11",
+]
+dynamic = ["version"]
+
+[tool.setuptools]
+dynamic = {"version" = {attr = 'bdfr.__version__'}}
+packages = ["bdfr", "bdfr.archive_entry", "bdfr.site_downloaders", "bdfr.site_downloaders.fallback_downloaders",]
+data-files = {"config" = ["bdfr/default_config.cfg",]}
+
+[project.optional-dependencies]
+dev = [
+    "black>=22.10.0",
+    "isort>=5.10.1",
+    "pre-commit>=2.20.0",
+    "pytest>=7.1.0",
+    "tox>=3.27.1",
+]
+
+[project.urls]
+"Homepage" = "https://aliparlakci.github.io/bulk-downloader-for-reddit"
+"Source" = "https://github.com/aliparlakci/bulk-downloader-for-reddit"
+"Bug Reports" = "https://github.com/aliparlakci/bulk-downloader-for-reddit/issues"
+
+[project.scripts]
+bdfr = "bdfr.__main__:cli"
+bdfr-archive = "bdfr.__main__:cli_archive"
+bdfr-clone = "bdfr.__main__:cli_clone"
+bdfr-download = "bdfr.__main__:cli_download"
+
 [tool.black]
 line-length = 120
 
 [tool.isort]
 profile = "black"
+py_version = 39
 multi_line_output = 3
 line_length = 120
+indent = 4
+
+[tool.pytest.ini_options]
+minversion = "7.1"
+addopts = "--strict-markers"
+testpaths = "tests"
+markers = [
+    "online: tests require a connection to the internet",
+    "reddit: tests require a connection to Reddit",
+    "slow: test is slow to run",
+    "authenticated: test requires an authenticated Reddit instance",
+    "testing: incomplete tests",
+]
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index 09df53c..0000000
--- a/pytest.ini
+++ /dev/null
@@ -1,7 +0,0 @@
-[pytest]
-addopts = --strict-markers
-markers =
-    online: tests require a connection to the internet
-    reddit: tests require a connection to Reddit
-    slow: test is slow to run
-    authenticated: test requires an authenticated Reddit instance
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 62e6925..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-appdirs>=1.4.4
-bs4>=0.0.1
-click>=7.1.2
-dict2xml>=1.7.0
-ffmpeg-python>=0.2.0
-praw>=7.2.0
-pyyaml>=5.4.1
-requests>=2.25.1
-yt-dlp>=2022.11.11
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index bb0ea60..0000000
--- a/setup.cfg
+++ /dev/null
@@ -1,26 +0,0 @@
-[metadata]
-name = bdfr
-description_file = README.md
-description_content_type = text/markdown
-home_page = https://github.com/aliparlakci/bulk-downloader-for-reddit
-keywords = reddit, download, archive
-version = 2.6.2
-author = Ali Parlakci
-author_email = parlakciali@gmail.com
-maintainer = Serene Arc
-maintainer_email = serenical@gmail.com
-license = GPLv3
-classifiers =
-    Programming Language :: Python :: 3
-    License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-    Natural Language :: English
-    Environment :: Console
-    Operating System :: OS Independent
-platforms = any
-
-[files]
-packages = bdfr
-
-[entry_points]
-console_scripts =
-    bdfr = bdfr.__main__:cli
diff --git a/setup.py b/setup.py
deleted file mode 100644
index c5518a6..0000000
--- a/setup.py
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env python3
-# encoding=utf-8
-
-from setuptools import setup
-
-setup(setup_requires=['pbr', 'appdirs'], pbr=True, data_files=[('config', ['bdfr/default_config.cfg'])], python_requires='>=3.9.0')
diff --git a/tox.ini b/tox.ini
index be451e6..01ece39 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,4 +1,6 @@
 [tox]
+requires =
+    tox>=3.27.1
 envlist =
     format
     format_check

From af6222e06c5f9b558ac391254844db6b28f4111f Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 17 Dec 2022 20:35:58 -0500
Subject: [PATCH 25/66] Cleanup tests

Cleans up test args on new tests.

Add log path to default config test so as not to mangle default log outside of tests.

Match setup functions to archive/clone.

Remove testing marker that was commited in error.
---
 pyproject.toml                                   |  1 -
 .../test_archive_integration.py                  | 10 ++--------
 .../integration_tests/test_clone_integration.py  | 10 ++--------
 .../test_download_integration.py                 | 16 +++++-----------
 4 files changed, 9 insertions(+), 28 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 23ae690..6879a6d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -78,5 +78,4 @@ markers = [
     "reddit: tests require a connection to Reddit",
     "slow: test is slow to run",
     "authenticated: test requires an authenticated Reddit instance",
-    "testing: incomplete tests",
 ]
diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py
index 1c0d30a..c5ad9fb 100644
--- a/tests/integration_tests/test_archive_integration.py
+++ b/tests/integration_tests/test_archive_integration.py
@@ -184,14 +184,8 @@ def test_cli_archive_soft_fail(test_args: list[str], tmp_path: Path):
 @pytest.mark.parametrize(
     ("test_args", "response"),
     (
-        (
-            ["--user", "nasa", "--submitted"],
-            502,
-        ),
-        (
-            ["--user", "nasa", "--submitted"],
-            504,
-        ),
+        (["--user", "nasa", "--submitted"], 502),
+        (["--user", "nasa", "--submitted"], 504),
     ),
 )
 def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
diff --git a/tests/integration_tests/test_clone_integration.py b/tests/integration_tests/test_clone_integration.py
index eb64364..cba4102 100644
--- a/tests/integration_tests/test_clone_integration.py
+++ b/tests/integration_tests/test_clone_integration.py
@@ -76,14 +76,8 @@ def test_cli_scrape_soft_fail(test_args: list[str], tmp_path: Path):
 @pytest.mark.parametrize(
     ("test_args", "response"),
     (
-        (
-            ["--user", "nasa", "--submitted"],
-            502,
-        ),
-        (
-            ["--user", "nasa", "--submitted"],
-            504,
-        ),
+        (["--user", "nasa", "--submitted"], 502),
+        (["--user", "nasa", "--submitted"], 504),
     ),
 )
 def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index e44c95e..287e8d4 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -15,7 +15,7 @@ does_test_config_exist = Path("./tests/test_config.cfg").exists()
 
 
 def copy_test_config(run_path: Path):
-    shutil.copy(Path("./tests/test_config.cfg"), Path(run_path, "./test_config.cfg"))
+    shutil.copy(Path("./tests/test_config.cfg"), Path(run_path, "test_config.cfg"))
 
 
 def create_basic_args_for_download_runner(test_args: list[str], run_path: Path):
@@ -25,7 +25,7 @@ def create_basic_args_for_download_runner(test_args: list[str], run_path: Path):
         str(run_path),
         "-v",
         "--config",
-        str(Path(run_path, "./test_config.cfg")),
+        str(Path(run_path, "test_config.cfg")),
         "--log",
         str(Path(run_path, "test_log.txt")),
     ] + test_args
@@ -279,7 +279,7 @@ def test_cli_download_hard_fail(test_args: list[str], tmp_path: Path):
 
 def test_cli_download_use_default_config(tmp_path: Path):
     runner = CliRunner()
-    test_args = ["download", "-vv", str(tmp_path)]
+    test_args = ["download", "-vv", str(tmp_path), "--log", str(Path(tmp_path, "test_log.txt"))]
     result = runner.invoke(cli, test_args)
     assert result.exit_code == 0
 
@@ -404,14 +404,8 @@ def test_cli_download_score_filter(test_args: list[str], was_filtered: bool, tmp
 @pytest.mark.parametrize(
     ("test_args", "response"),
     (
-        (
-            ["--user", "nasa", "--submitted"],
-            502,
-        ),
-        (
-            ["--user", "nasa", "--submitted"],
-            504,
-        ),
+        (["--user", "nasa", "--submitted"], 502),
+        (["--user", "nasa", "--submitted"], 504),
     ),
 )
 def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):

From 66049a402185a117297e9ec3daa9100c8ffe4fa2 Mon Sep 17 00:00:00 2001
From: ZapperDJ <zapperdj@gmail.com>
Date: Mon, 19 Dec 2022 16:54:09 +0100
Subject: [PATCH 26/66] Add documentation for unsavepost.py

---
 README.md         |  4 ++++
 scripts/README.md | 21 +++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/README.md b/README.md
index 2c245c6..204e76f 100644
--- a/README.md
+++ b/README.md
@@ -391,6 +391,10 @@ The way to fix this is to use the `--log` option to manually specify where the l
 
 The logfiles that the BDFR outputs are consistent and quite detailed and in a format that is amenable to regex. To this end, a number of bash scripts have been [included here](./scripts). They show examples for how to extract successfully downloaded IDs, failed IDs, and more besides.
 
+## Unsaving posts
+
+Back in v1 there was an option to unsave posts from your account when downloading, but it was removed from the core BDFR on v2 as it is considered a read-only tool. However, for those missing this functionality, a script was created that uses the log files to achieve this. There is info on how to use this on the README.md file on the scripts subdirectory.
+
 ## List of currently supported sources
 
   - Direct links (links leading to a file)
diff --git a/scripts/README.md b/scripts/README.md
index 4bb098b..b6e7c80 100644
--- a/scripts/README.md
+++ b/scripts/README.md
@@ -6,6 +6,7 @@ Due to the verboseness of the logs, a great deal of information can be gathered
   - [Script to extract all failed download IDs](#extract-all-failed-ids)
   - [Timestamp conversion](#converting-bdfrv1-timestamps-to-bdfrv2-timestamps)
   - [Printing summary statistics for a run](#printing-summary-statistics)
+  - [Unsaving posts from your account after downloading](#unsave-posts-after-downloading)
 
 ## Extract all Successfully Downloaded IDs
 
@@ -67,3 +68,23 @@ Excluded submissions: 1146
 Files with existing hash skipped: 0
 Submissions from excluded subreddits: 0
 ```
+
+## Unsave Posts After Downloading
+
+[This script](unsaveposts.py) takes a list of submission IDs from a file named `successfulids` created with the `extract_successful_ids.sh` script and unsaves them from your account. To make it work you will need to make a user script in your reddit profile like this:
+ - Fill in the username and password fields in the script. Make sure you keep the quotes around the fields.  
+ - Go to https://old.reddit.com/prefs/apps/
+ - Click on `Develop an app` at the bottom.
+ - Make sure you select a `script` not a `web app`.
+ - Name it `Unsave Posts`.
+ - Fill in the `Redirect URI` field with `127.0.0.0`.
+ - Save it.
+ - Fill in the `client_id` and `client_secret` fields on the script. The client ID is the 14 character string under the name you gave your script. .It'll look like a bunch of random characters like this: pspYLwDoci9z_A. The client secret is the longer string next to "secret". Again keep the quotes around the fields.
+
+Now the script is ready tu run. Just execute it like this:
+
+```bash
+python3.9 -m bdfr download DOWNLOAD_DIR --authenticate --user me --saved --log LOGFILE_LOCATION
+./extract_successful_ids.sh LOGFILE_LOCATION > successfulids
+./unsaveposts.py
+```

From 603e7de04d9fe4d5717629b2606fe1b454615353 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Mon, 19 Dec 2022 11:02:06 -0500
Subject: [PATCH 27/66] Redgifs fix

Handle redgifs link with trailing / causing id to return empty string.
---
 bdfr/site_downloaders/redgifs.py       | 2 ++
 tests/site_downloaders/test_redgifs.py | 1 +
 2 files changed, 3 insertions(+)

diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py
index 625cf7d..84fb3c3 100644
--- a/bdfr/site_downloaders/redgifs.py
+++ b/bdfr/site_downloaders/redgifs.py
@@ -24,6 +24,8 @@ class Redgifs(BaseDownloader):
     @staticmethod
     def _get_link(url: str) -> set[str]:
         try:
+            if url.endswith("/"):
+                url = url.removesuffix("/")
             redgif_id = re.match(r".*/(.*?)(\..{0,})?$", url).group(1)
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Redgifs ID from {url}")
diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py
index 0e1a497..bfe683f 100644
--- a/tests/site_downloaders/test_redgifs.py
+++ b/tests/site_downloaders/test_redgifs.py
@@ -29,6 +29,7 @@ from bdfr.site_downloaders.redgifs import Redgifs
                 "UnripeUnkemptWoodpecker-large.jpg",
             },
         ),
+        ("https://www.redgifs.com/watch/genuineprivateguillemot/", {"GenuinePrivateGuillemot.mp4"}),
     ),
 )
 def test_get_link(test_url: str, expected: set[str]):

From 2e2dfe671b78b00c695957ee0ea1844afb5fe446 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 19 Dec 2022 17:33:07 -0500
Subject: [PATCH 28/66] Fix fish/zsh completions

fixes mistake in fish/zsh completions.
---
 bdfr/completion.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/bdfr/completion.py b/bdfr/completion.py
index 8f4f122..fac944f 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -26,16 +26,16 @@ class Completion:
             comp_dir = self.share_dir + "/fish/vendor_completions.d/"
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "fish_source"
-                with open(comp_dir + point, "w") as file:
+                with open(comp_dir + point + ".fish", "w") as file:
                     file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
-                    print(f"Fish completion for {point} written to {comp_dir}{point}")
+                    print(f"Fish completion for {point} written to {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
             comp_dir = self.share_dir + "/zsh/site-functions/"
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "zsh_source"
-                with open(comp_dir + point, "w") as file:
+                with open(comp_dir + "_" + point, "w") as file:
                     file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
-                    print(f"Zsh completion for {point} written to {comp_dir}{point}")
+                    print(f"Zsh completion for {point} written to {comp_dir}_{point}")
 
     def uninstall(self):
         if self.shell in ("all", "bash"):
@@ -47,12 +47,12 @@ class Completion:
         if self.shell in ("all", "fish"):
             comp_dir = self.share_dir + "/fish/vendor_completions.d/"
             for point in self.entry_points:
-                if os.path.exists(comp_dir + point):
-                    os.remove(comp_dir + point)
-                    print(f"Fish completion for {point} removed from {comp_dir}{point}")
+                if os.path.exists(comp_dir + point + ".fish"):
+                    os.remove(comp_dir + point + ".fish")
+                    print(f"Fish completion for {point} removed from {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
             comp_dir = self.share_dir + "/zsh/site-functions/"
             for point in self.entry_points:
-                if os.path.exists(comp_dir + point):
-                    os.remove(comp_dir + point)
-                    print(f"Zsh completion for {point} removed from {comp_dir}{point}")
+                if os.path.exists(comp_dir + "_" + point):
+                    os.remove(comp_dir + "_" + point)
+                    print(f"Zsh completion for {point} removed from {comp_dir}_{point}")

From 5d3a539eda7edf77c6339b1ee664f674c3884920 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 19 Dec 2022 17:54:34 -0500
Subject: [PATCH 29/66] Fix install of completion if dirs missing

Fixes situations if completion directories are missing and adds tests for installer.
---
 bdfr/completion.py       |  9 ++++++++
 tests/test_completion.py | 50 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 59 insertions(+)
 create mode 100644 tests/test_completion.py

diff --git a/bdfr/completion.py b/bdfr/completion.py
index fac944f..43a9743 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -17,6 +17,9 @@ class Completion:
     def install(self):
         if self.shell in ("all", "bash"):
             comp_dir = self.share_dir + "/bash-completion/completions/"
+            if not os.path.exists(comp_dir):
+                print("Creating Bash completion directory.")
+                os.makedirs(comp_dir, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "bash_source"
                 with open(comp_dir + point, "w") as file:
@@ -24,6 +27,9 @@ class Completion:
                     print(f"Bash completion for {point} written to {comp_dir}{point}")
         if self.shell in ("all", "fish"):
             comp_dir = self.share_dir + "/fish/vendor_completions.d/"
+            if not os.path.exists(comp_dir):
+                print("Creating Fish completion directory.")
+                os.makedirs(comp_dir, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "fish_source"
                 with open(comp_dir + point + ".fish", "w") as file:
@@ -31,6 +37,9 @@ class Completion:
                     print(f"Fish completion for {point} written to {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
             comp_dir = self.share_dir + "/zsh/site-functions/"
+            if not os.path.exists(comp_dir):
+                print("Creating Zsh completion directory.")
+                os.makedirs(comp_dir, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "zsh_source"
                 with open(comp_dir + "_" + point, "w") as file:
diff --git a/tests/test_completion.py b/tests/test_completion.py
new file mode 100644
index 0000000..91f9fd2
--- /dev/null
+++ b/tests/test_completion.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from bdfr.completion import Completion
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
+def test_cli_completion_all(tmp_path: Path):
+    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+        Completion("all").install()
+        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 1
+        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 1
+        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 1
+        Completion("all").uninstall()
+        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 0
+        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 0
+        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 0
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
+def test_cli_completion_bash(tmp_path: Path):
+    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+        Completion("bash").install()
+        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 1
+        Completion("bash").uninstall()
+        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 0
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
+def test_cli_completion_fish(tmp_path: Path):
+    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+        Completion("fish").install()
+        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 1
+        Completion("fish").uninstall()
+        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 0
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
+def test_cli_completion_zsh(tmp_path: Path):
+    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+        Completion("zsh").install()
+        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 1
+        Completion("zsh").uninstall()
+        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 0

From 83f45e7f60e7fd6ff5d1156af0ebf28904515ac8 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 19 Dec 2022 18:32:37 -0500
Subject: [PATCH 30/66] Standardize shebang and coding declaration

Standardizes shebang and coding declarations.

Coding matches what's used by install tools such as pip(x).

Removes a few init files that were not needed.
---
 bdfr/__init__.py                                              | 3 +++
 bdfr/__main__.py                                              | 1 +
 bdfr/archive_entry/__init__.py                                | 2 --
 bdfr/archive_entry/base_archive_entry.py                      | 2 +-
 bdfr/archive_entry/comment_archive_entry.py                   | 2 +-
 bdfr/archive_entry/submission_archive_entry.py                | 2 +-
 bdfr/archiver.py                                              | 2 +-
 bdfr/cloner.py                                                | 2 +-
 bdfr/completion.py                                            | 2 +-
 bdfr/configuration.py                                         | 2 +-
 bdfr/connector.py                                             | 2 +-
 bdfr/download_filter.py                                       | 2 +-
 bdfr/downloader.py                                            | 2 +-
 bdfr/exceptions.py                                            | 3 ++-
 bdfr/file_name_formatter.py                                   | 3 ++-
 bdfr/oauth2.py                                                | 2 +-
 bdfr/resource.py                                              | 2 +-
 bdfr/site_authenticator.py                                    | 2 +-
 bdfr/site_downloaders/__init__.py                             | 0
 bdfr/site_downloaders/base_downloader.py                      | 2 +-
 bdfr/site_downloaders/delay_for_reddit.py                     | 1 +
 bdfr/site_downloaders/direct.py                               | 1 +
 bdfr/site_downloaders/download_factory.py                     | 2 +-
 bdfr/site_downloaders/erome.py                                | 1 +
 bdfr/site_downloaders/fallback_downloaders/__init__.py        | 0
 .../fallback_downloaders/fallback_downloader.py               | 2 +-
 bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py  | 2 +-
 bdfr/site_downloaders/gallery.py                              | 1 +
 bdfr/site_downloaders/gfycat.py                               | 1 +
 bdfr/site_downloaders/imgur.py                                | 1 +
 bdfr/site_downloaders/pornhub.py                              | 2 +-
 bdfr/site_downloaders/redgifs.py                              | 1 +
 bdfr/site_downloaders/self_post.py                            | 1 +
 bdfr/site_downloaders/vidble.py                               | 3 ++-
 bdfr/site_downloaders/vreddit.py                              | 1 +
 bdfr/site_downloaders/youtube.py                              | 1 +
 tests/__init__.py                                             | 0
 tests/archive_entry/__init__.py                               | 2 --
 tests/archive_entry/test_comment_archive_entry.py             | 2 +-
 tests/archive_entry/test_submission_archive_entry.py          | 2 +-
 tests/conftest.py                                             | 2 +-
 tests/integration_tests/__init__.py                           | 2 --
 tests/integration_tests/test_archive_integration.py           | 2 +-
 tests/integration_tests/test_clone_integration.py             | 2 +-
 tests/integration_tests/test_download_integration.py          | 2 +-
 tests/site_downloaders/__init__.py                            | 0
 tests/site_downloaders/fallback_downloaders/__init__.py       | 0
 .../fallback_downloaders/test_ytdlp_fallback.py               | 1 +
 tests/site_downloaders/test_delay_for_reddit.py               | 2 +-
 tests/site_downloaders/test_direct.py                         | 2 +-
 tests/site_downloaders/test_download_factory.py               | 2 +-
 tests/site_downloaders/test_erome.py                          | 3 ++-
 tests/site_downloaders/test_gallery.py                        | 2 +-
 tests/site_downloaders/test_gfycat.py                         | 2 +-
 tests/site_downloaders/test_imgur.py                          | 2 +-
 tests/site_downloaders/test_pornhub.py                        | 2 +-
 tests/site_downloaders/test_redgifs.py                        | 2 +-
 tests/site_downloaders/test_self_post.py                      | 2 +-
 tests/site_downloaders/test_vidble.py                         | 3 ++-
 tests/site_downloaders/test_vreddit.py                        | 2 +-
 tests/site_downloaders/test_youtube.py                        | 2 +-
 tests/test_archiver.py                                        | 2 +-
 tests/test_configuration.py                                   | 2 +-
 tests/test_connector.py                                       | 3 ++-
 tests/test_download_filter.py                                 | 2 +-
 tests/test_downloader.py                                      | 2 +-
 tests/test_file_name_formatter.py                             | 4 ++--
 tests/test_oauth2.py                                          | 2 +-
 tests/test_resource.py                                        | 2 +-
 69 files changed, 70 insertions(+), 55 deletions(-)
 delete mode 100644 bdfr/archive_entry/__init__.py
 delete mode 100644 bdfr/site_downloaders/__init__.py
 delete mode 100644 bdfr/site_downloaders/fallback_downloaders/__init__.py
 delete mode 100644 tests/__init__.py
 delete mode 100644 tests/archive_entry/__init__.py
 delete mode 100644 tests/integration_tests/__init__.py
 delete mode 100644 tests/site_downloaders/__init__.py
 delete mode 100644 tests/site_downloaders/fallback_downloaders/__init__.py

diff --git a/bdfr/__init__.py b/bdfr/__init__.py
index b482efe..6bcee53 100644
--- a/bdfr/__init__.py
+++ b/bdfr/__init__.py
@@ -1 +1,4 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
 __version__ = "2.6.2"
diff --git a/bdfr/__main__.py b/bdfr/__main__.py
index 57373c9..e35ba0a 100644
--- a/bdfr/__main__.py
+++ b/bdfr/__main__.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 import sys
diff --git a/bdfr/archive_entry/__init__.py b/bdfr/archive_entry/__init__.py
deleted file mode 100644
index d4c1799..0000000
--- a/bdfr/archive_entry/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-# coding=utf-8
diff --git a/bdfr/archive_entry/base_archive_entry.py b/bdfr/archive_entry/base_archive_entry.py
index 49ea58a..3dea5e4 100644
--- a/bdfr/archive_entry/base_archive_entry.py
+++ b/bdfr/archive_entry/base_archive_entry.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from abc import ABC, abstractmethod
 from typing import Union
diff --git a/bdfr/archive_entry/comment_archive_entry.py b/bdfr/archive_entry/comment_archive_entry.py
index 1c72811..cc59373 100644
--- a/bdfr/archive_entry/comment_archive_entry.py
+++ b/bdfr/archive_entry/comment_archive_entry.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 
diff --git a/bdfr/archive_entry/submission_archive_entry.py b/bdfr/archive_entry/submission_archive_entry.py
index 92f326e..38f1d34 100644
--- a/bdfr/archive_entry/submission_archive_entry.py
+++ b/bdfr/archive_entry/submission_archive_entry.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 
diff --git a/bdfr/archiver.py b/bdfr/archiver.py
index e2ed33d..be5a445 100644
--- a/bdfr/archiver.py
+++ b/bdfr/archiver.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import json
 import logging
diff --git a/bdfr/cloner.py b/bdfr/cloner.py
index e82cfaa..53108c0 100644
--- a/bdfr/cloner.py
+++ b/bdfr/cloner.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 from time import sleep
diff --git a/bdfr/completion.py b/bdfr/completion.py
index 43a9743..9c7d6b2 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import os
 import subprocess
diff --git a/bdfr/configuration.py b/bdfr/configuration.py
index a2a5310..0d00192 100644
--- a/bdfr/configuration.py
+++ b/bdfr/configuration.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 from argparse import Namespace
diff --git a/bdfr/connector.py b/bdfr/connector.py
index e5d74a2..bf50f32 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import configparser
 import importlib.resources
diff --git a/bdfr/download_filter.py b/bdfr/download_filter.py
index 9019cc9..0def316 100644
--- a/bdfr/download_filter.py
+++ b/bdfr/download_filter.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 import re
diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index 1cb6d46..7ad8a6b 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import hashlib
 import logging.handlers
diff --git a/bdfr/exceptions.py b/bdfr/exceptions.py
index 1757cd9..e7e4415 100644
--- a/bdfr/exceptions.py
+++ b/bdfr/exceptions.py
@@ -1,4 +1,5 @@
-#!/usr/bin/env
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 
 class BulkDownloaderException(Exception):
diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py
index 684c626..9ee481d 100644
--- a/bdfr/file_name_formatter.py
+++ b/bdfr/file_name_formatter.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
+
 import datetime
 import logging
 import platform
diff --git a/bdfr/oauth2.py b/bdfr/oauth2.py
index 60f2169..28b956a 100644
--- a/bdfr/oauth2.py
+++ b/bdfr/oauth2.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import configparser
 import logging
diff --git a/bdfr/resource.py b/bdfr/resource.py
index 0f5404c..bd3ae88 100644
--- a/bdfr/resource.py
+++ b/bdfr/resource.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import hashlib
 import logging
diff --git a/bdfr/site_authenticator.py b/bdfr/site_authenticator.py
index bbf3b46..08b98e0 100644
--- a/bdfr/site_authenticator.py
+++ b/bdfr/site_authenticator.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import configparser
 
diff --git a/bdfr/site_downloaders/__init__.py b/bdfr/site_downloaders/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/bdfr/site_downloaders/base_downloader.py b/bdfr/site_downloaders/base_downloader.py
index f3ecec5..e4ac111 100644
--- a/bdfr/site_downloaders/base_downloader.py
+++ b/bdfr/site_downloaders/base_downloader.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 from abc import ABC, abstractmethod
diff --git a/bdfr/site_downloaders/delay_for_reddit.py b/bdfr/site_downloaders/delay_for_reddit.py
index 3380731..40a7f9b 100644
--- a/bdfr/site_downloaders/delay_for_reddit.py
+++ b/bdfr/site_downloaders/delay_for_reddit.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 from typing import Optional
diff --git a/bdfr/site_downloaders/direct.py b/bdfr/site_downloaders/direct.py
index 4a6ac92..061ad7f 100644
--- a/bdfr/site_downloaders/direct.py
+++ b/bdfr/site_downloaders/direct.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 from typing import Optional
 
diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index 638316f..d44be21 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import re
 import urllib.parse
diff --git a/bdfr/site_downloaders/erome.py b/bdfr/site_downloaders/erome.py
index 26469bc..bf139d2 100644
--- a/bdfr/site_downloaders/erome.py
+++ b/bdfr/site_downloaders/erome.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 import re
diff --git a/bdfr/site_downloaders/fallback_downloaders/__init__.py b/bdfr/site_downloaders/fallback_downloaders/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py b/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py
index 3bc615d..124724a 100644
--- a/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py
+++ b/bdfr/site_downloaders/fallback_downloaders/fallback_downloader.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from abc import ABC, abstractmethod
 
diff --git a/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
index 900c8e9..41f8474 100644
--- a/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
+++ b/bdfr/site_downloaders/fallback_downloaders/ytdlp_fallback.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 from typing import Optional
diff --git a/bdfr/site_downloaders/gallery.py b/bdfr/site_downloaders/gallery.py
index 278932f..6f00410 100644
--- a/bdfr/site_downloaders/gallery.py
+++ b/bdfr/site_downloaders/gallery.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 from typing import Optional
diff --git a/bdfr/site_downloaders/gfycat.py b/bdfr/site_downloaders/gfycat.py
index 7862d33..d7c60ca 100644
--- a/bdfr/site_downloaders/gfycat.py
+++ b/bdfr/site_downloaders/gfycat.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import json
 import re
diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index 0b9ecdd..a4c378f 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import json
 import re
diff --git a/bdfr/site_downloaders/pornhub.py b/bdfr/site_downloaders/pornhub.py
index db37720..8ce4492 100644
--- a/bdfr/site_downloaders/pornhub.py
+++ b/bdfr/site_downloaders/pornhub.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import logging
 from typing import Optional
diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py
index 625cf7d..3144c22 100644
--- a/bdfr/site_downloaders/redgifs.py
+++ b/bdfr/site_downloaders/redgifs.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import json
 import re
diff --git a/bdfr/site_downloaders/self_post.py b/bdfr/site_downloaders/self_post.py
index 1b76b92..5719e59 100644
--- a/bdfr/site_downloaders/self_post.py
+++ b/bdfr/site_downloaders/self_post.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 from typing import Optional
diff --git a/bdfr/site_downloaders/vidble.py b/bdfr/site_downloaders/vidble.py
index a79ee25..aa1e949 100644
--- a/bdfr/site_downloaders/vidble.py
+++ b/bdfr/site_downloaders/vidble.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
+
 import itertools
 import logging
 import re
diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py
index a71d350..8f6022e 100644
--- a/bdfr/site_downloaders/vreddit.py
+++ b/bdfr/site_downloaders/vreddit.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 import tempfile
diff --git a/bdfr/site_downloaders/youtube.py b/bdfr/site_downloaders/youtube.py
index f4f8622..f0c0677 100644
--- a/bdfr/site_downloaders/youtube.py
+++ b/bdfr/site_downloaders/youtube.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 import logging
 import tempfile
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/archive_entry/__init__.py b/tests/archive_entry/__init__.py
deleted file mode 100644
index d4c1799..0000000
--- a/tests/archive_entry/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-# coding=utf-8
diff --git a/tests/archive_entry/test_comment_archive_entry.py b/tests/archive_entry/test_comment_archive_entry.py
index 8e6f224..1895a89 100644
--- a/tests/archive_entry/test_comment_archive_entry.py
+++ b/tests/archive_entry/test_comment_archive_entry.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import praw
 import pytest
diff --git a/tests/archive_entry/test_submission_archive_entry.py b/tests/archive_entry/test_submission_archive_entry.py
index 666eec3..8b83f1d 100644
--- a/tests/archive_entry/test_submission_archive_entry.py
+++ b/tests/archive_entry/test_submission_archive_entry.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import praw
 import pytest
diff --git a/tests/conftest.py b/tests/conftest.py
index 3f871a3..77a26fb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import configparser
 import socket
diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py
deleted file mode 100644
index d4c1799..0000000
--- a/tests/integration_tests/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-# coding=utf-8
diff --git a/tests/integration_tests/test_archive_integration.py b/tests/integration_tests/test_archive_integration.py
index c5ad9fb..42689a8 100644
--- a/tests/integration_tests/test_archive_integration.py
+++ b/tests/integration_tests/test_archive_integration.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import re
 import shutil
diff --git a/tests/integration_tests/test_clone_integration.py b/tests/integration_tests/test_clone_integration.py
index cba4102..60e4012 100644
--- a/tests/integration_tests/test_clone_integration.py
+++ b/tests/integration_tests/test_clone_integration.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import shutil
 from pathlib import Path
diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 287e8d4..138ea61 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import shutil
 from pathlib import Path
diff --git a/tests/site_downloaders/__init__.py b/tests/site_downloaders/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/site_downloaders/fallback_downloaders/__init__.py b/tests/site_downloaders/fallback_downloaders/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py
index 9823d08..b735539 100644
--- a/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py
+++ b/tests/site_downloaders/fallback_downloaders/test_ytdlp_fallback.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/site_downloaders/test_delay_for_reddit.py b/tests/site_downloaders/test_delay_for_reddit.py
index 65d080c..045c022 100644
--- a/tests/site_downloaders/test_delay_for_reddit.py
+++ b/tests/site_downloaders/test_delay_for_reddit.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import Mock
 
diff --git a/tests/site_downloaders/test_direct.py b/tests/site_downloaders/test_direct.py
index b652d9a..14190ee 100644
--- a/tests/site_downloaders/test_direct.py
+++ b/tests/site_downloaders/test_direct.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import Mock
 
diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py
index 581656d..2dc3a06 100644
--- a/tests/site_downloaders/test_download_factory.py
+++ b/tests/site_downloaders/test_download_factory.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import praw
 import pytest
diff --git a/tests/site_downloaders/test_erome.py b/tests/site_downloaders/test_erome.py
index 1baeb66..ce32e88 100644
--- a/tests/site_downloaders/test_erome.py
+++ b/tests/site_downloaders/test_erome.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
+
 import re
 from unittest.mock import MagicMock
 
diff --git a/tests/site_downloaders/test_gallery.py b/tests/site_downloaders/test_gallery.py
index 57d055b..c3cc86f 100644
--- a/tests/site_downloaders/test_gallery.py
+++ b/tests/site_downloaders/test_gallery.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import praw
 import pytest
diff --git a/tests/site_downloaders/test_gfycat.py b/tests/site_downloaders/test_gfycat.py
index d436636..0cfb36f 100644
--- a/tests/site_downloaders/test_gfycat.py
+++ b/tests/site_downloaders/test_gfycat.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import Mock
 
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 6b49cd5..7f587c9 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import Mock
 
diff --git a/tests/site_downloaders/test_pornhub.py b/tests/site_downloaders/test_pornhub.py
index 42ca5a0..d9971cb 100644
--- a/tests/site_downloaders/test_pornhub.py
+++ b/tests/site_downloaders/test_pornhub.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py
index 0e1a497..6f5ce50 100644
--- a/tests/site_downloaders/test_redgifs.py
+++ b/tests/site_downloaders/test_redgifs.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import re
 from unittest.mock import Mock
diff --git a/tests/site_downloaders/test_self_post.py b/tests/site_downloaders/test_self_post.py
index 104fb3b..9574b3c 100644
--- a/tests/site_downloaders/test_self_post.py
+++ b/tests/site_downloaders/test_self_post.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import praw
 import pytest
diff --git a/tests/site_downloaders/test_vidble.py b/tests/site_downloaders/test_vidble.py
index 16b5a3b..41398e7 100644
--- a/tests/site_downloaders/test_vidble.py
+++ b/tests/site_downloaders/test_vidble.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
+
 from unittest.mock import Mock
 
 import pytest
diff --git a/tests/site_downloaders/test_vreddit.py b/tests/site_downloaders/test_vreddit.py
index 6e79ba0..d5cc121 100644
--- a/tests/site_downloaders/test_vreddit.py
+++ b/tests/site_downloaders/test_vreddit.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/site_downloaders/test_youtube.py b/tests/site_downloaders/test_youtube.py
index 7a45a3c..3100215 100644
--- a/tests/site_downloaders/test_youtube.py
+++ b/tests/site_downloaders/test_youtube.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/test_archiver.py b/tests/test_archiver.py
index 932a2ab..cdd12d0 100644
--- a/tests/test_archiver.py
+++ b/tests/test_archiver.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from pathlib import Path
 from unittest.mock import MagicMock
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index 652c401..e7999b3 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/test_connector.py b/tests/test_connector.py
index 01b6a92..9681e4b 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
+
 from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Iterator
diff --git a/tests/test_download_filter.py b/tests/test_download_filter.py
index 07b7d67..3b4d6b8 100644
--- a/tests/test_download_filter.py
+++ b/tests/test_download_filter.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index ba81b80..d7aa8dd 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import os
 import re
diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py
index c04e07d..4964b3b 100644
--- a/tests/test_file_name_formatter.py
+++ b/tests/test_file_name_formatter.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import platform
 import sys
@@ -46,7 +46,7 @@ def do_test_path_equality(result: Path, expected: str) -> bool:
         expected = Path(*expected)
     else:
         expected = Path(expected)
-    return str(result).endswith(str(expected))
+    return str(result).endswith(str(expected))  # noqa: FURB123
 
 
 @pytest.fixture(scope="session")
diff --git a/tests/test_oauth2.py b/tests/test_oauth2.py
index 3014c37..123f750 100644
--- a/tests/test_oauth2.py
+++ b/tests/test_oauth2.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import configparser
 from pathlib import Path
diff --git a/tests/test_resource.py b/tests/test_resource.py
index 146d9a0..e17d16a 100644
--- a/tests/test_resource.py
+++ b/tests/test_resource.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 from unittest.mock import MagicMock
 

From b3e477720667bbf5a39c21dc14b62852fd96465b Mon Sep 17 00:00:00 2001
From: OMEGA_RAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 19 Dec 2022 22:02:16 -0500
Subject: [PATCH 31/66] Update download_factory.py

Attempt to fix #724

Narrows down characters available to extensions in the regex. Outside of  3 and 4, the only extensions that I can think of this doesn't hit are bz2 and 7z (which wasn't caught before).
---
 bdfr/site_downloaders/download_factory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index 638316f..1bc7507 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -30,7 +30,7 @@ class DownloadFactory:
             return Imgur
         elif re.match(r"(i\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
             return Redgifs
-        elif re.match(r".*/.*\.\w{3,4}(\?[\w;&=]*)?$", sanitised_url) and not DownloadFactory.is_web_resource(
+        elif re.match(r".*/.*\.[a-zA-Z34]{3,4}(\?[\w;&=]*)?$", sanitised_url) and not DownloadFactory.is_web_resource(
             sanitised_url
         ):
             return Direct

From da74096cdec9561593c04f1a39656c916ec10943 Mon Sep 17 00:00:00 2001
From: OMEGA_RAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 19 Dec 2022 22:04:49 -0500
Subject: [PATCH 32/66] Update test_download_factory.py

---
 tests/site_downloaders/test_download_factory.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py
index 581656d..b369486 100644
--- a/tests/site_downloaders/test_download_factory.py
+++ b/tests/site_downloaders/test_download_factory.py
@@ -65,6 +65,7 @@ def test_factory_lever_good(test_submission_url: str, expected_class: BaseDownlo
         "https://www.google.com",
         "https://www.google.com/test",
         "https://www.google.com/test/",
+        "https://www.tiktok.com/@keriberry.420",
     ),
 )
 def test_factory_lever_bad(test_url: str):

From 57ac0130a62eca0176e6d3a2bebe45f81a46558a Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Tue, 20 Dec 2022 12:16:31 -0500
Subject: [PATCH 33/66] revert init files

---
 README.md                                               | 4 +++-
 bdfr/archive_entry/__init__.py                          | 2 ++
 bdfr/site_downloaders/__init__.py                       | 2 ++
 bdfr/site_downloaders/fallback_downloaders/__init__.py  | 2 ++
 tests/__init__.py                                       | 2 ++
 tests/archive_entry/__init__.py                         | 2 ++
 tests/integration_tests/__init__.py                     | 2 ++
 tests/site_downloaders/__init__.py                      | 2 ++
 tests/site_downloaders/fallback_downloaders/__init__.py | 2 ++
 9 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 bdfr/archive_entry/__init__.py
 create mode 100644 bdfr/site_downloaders/__init__.py
 create mode 100644 bdfr/site_downloaders/fallback_downloaders/__init__.py
 create mode 100644 tests/__init__.py
 create mode 100644 tests/archive_entry/__init__.py
 create mode 100644 tests/integration_tests/__init__.py
 create mode 100644 tests/site_downloaders/__init__.py
 create mode 100644 tests/site_downloaders/fallback_downloaders/__init__.py

diff --git a/README.md b/README.md
index f732f6e..120448e 100644
--- a/README.md
+++ b/README.md
@@ -115,7 +115,9 @@ would be equilavent to (take note that in YAML there is `file_scheme` instead of
 bdfr download ./path/to/output --skip mp4 --skip avi --file-scheme "{UPVOTES}_{REDDITOR}_{POSTID}_{DATE}" -L 10 -S top --subreddit EarthPorn --subreddit CityPorn
 ```
 
-In case when the same option is specified both in the YAML file and in as a command line argument, the command line argument takes prs
+Any option that can be specified multiple times should be formatted like subreddit is above.
+
+In case when the same option is specified both in the YAML file and in as a command line argument, the command line argument takes priority
 
 ## Options
 
diff --git a/bdfr/archive_entry/__init__.py b/bdfr/archive_entry/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/bdfr/archive_entry/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/bdfr/site_downloaders/__init__.py b/bdfr/site_downloaders/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/bdfr/site_downloaders/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/bdfr/site_downloaders/fallback_downloaders/__init__.py b/bdfr/site_downloaders/fallback_downloaders/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/bdfr/site_downloaders/fallback_downloaders/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/tests/archive_entry/__init__.py b/tests/archive_entry/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/tests/archive_entry/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/tests/integration_tests/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/tests/site_downloaders/__init__.py b/tests/site_downloaders/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/tests/site_downloaders/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
diff --git a/tests/site_downloaders/fallback_downloaders/__init__.py b/tests/site_downloaders/fallback_downloaders/__init__.py
new file mode 100644
index 0000000..56fafa5
--- /dev/null
+++ b/tests/site_downloaders/fallback_downloaders/__init__.py
@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-

From 2aea7d0d489299dd91cc4c335add67c177fcefcc Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Tue, 20 Dec 2022 13:05:50 -0500
Subject: [PATCH 34/66] Move completion to pathlib

---
 bdfr/completion.py       | 29 +++++++++++++++--------------
 tests/test_completion.py | 36 ++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 30 deletions(-)

diff --git a/bdfr/completion.py b/bdfr/completion.py
index 43a9743..668fb2a 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 # coding=utf-8
 
-import os
 import subprocess
+from os import environ
+from pathlib import Path
 
 import appdirs
 
@@ -10,16 +11,16 @@ import appdirs
 class Completion:
     def __init__(self, shell: str):
         self.shell = shell
-        self.env = os.environ.copy()
+        self.env = environ.copy()
         self.share_dir = appdirs.user_data_dir()
         self.entry_points = ["bdfr", "bdfr-archive", "bdfr-clone", "bdfr-download"]
 
     def install(self):
         if self.shell in ("all", "bash"):
             comp_dir = self.share_dir + "/bash-completion/completions/"
-            if not os.path.exists(comp_dir):
+            if not Path(comp_dir).exists():
                 print("Creating Bash completion directory.")
-                os.makedirs(comp_dir, exist_ok=True)
+                Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "bash_source"
                 with open(comp_dir + point, "w") as file:
@@ -27,9 +28,9 @@ class Completion:
                     print(f"Bash completion for {point} written to {comp_dir}{point}")
         if self.shell in ("all", "fish"):
             comp_dir = self.share_dir + "/fish/vendor_completions.d/"
-            if not os.path.exists(comp_dir):
+            if not Path(comp_dir).exists():
                 print("Creating Fish completion directory.")
-                os.makedirs(comp_dir, exist_ok=True)
+                Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "fish_source"
                 with open(comp_dir + point + ".fish", "w") as file:
@@ -37,9 +38,9 @@ class Completion:
                     print(f"Fish completion for {point} written to {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
             comp_dir = self.share_dir + "/zsh/site-functions/"
-            if not os.path.exists(comp_dir):
+            if not Path(comp_dir).exists():
                 print("Creating Zsh completion directory.")
-                os.makedirs(comp_dir, exist_ok=True)
+                Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "zsh_source"
                 with open(comp_dir + "_" + point, "w") as file:
@@ -50,18 +51,18 @@ class Completion:
         if self.shell in ("all", "bash"):
             comp_dir = self.share_dir + "/bash-completion/completions/"
             for point in self.entry_points:
-                if os.path.exists(comp_dir + point):
-                    os.remove(comp_dir + point)
+                if Path(comp_dir + point).exists():
+                    Path(comp_dir + point).unlink()
                     print(f"Bash completion for {point} removed from {comp_dir}{point}")
         if self.shell in ("all", "fish"):
             comp_dir = self.share_dir + "/fish/vendor_completions.d/"
             for point in self.entry_points:
-                if os.path.exists(comp_dir + point + ".fish"):
-                    os.remove(comp_dir + point + ".fish")
+                if Path(comp_dir + point + ".fish").exists():
+                    Path(comp_dir + point + ".fish").unlink()
                     print(f"Fish completion for {point} removed from {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
             comp_dir = self.share_dir + "/zsh/site-functions/"
             for point in self.entry_points:
-                if os.path.exists(comp_dir + "_" + point):
-                    os.remove(comp_dir + "_" + point)
+                if Path(comp_dir + "_" + point).exists():
+                    Path(comp_dir + "_" + point).unlink()
                     print(f"Zsh completion for {point} removed from {comp_dir}_{point}")
diff --git a/tests/test_completion.py b/tests/test_completion.py
index 91f9fd2..e29682a 100644
--- a/tests/test_completion.py
+++ b/tests/test_completion.py
@@ -12,39 +12,43 @@ from bdfr.completion import Completion
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
 def test_cli_completion_all(tmp_path: Path):
-    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+    tmp_path = str(tmp_path)
+    with patch("appdirs.user_data_dir", return_value=tmp_path):
         Completion("all").install()
-        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 1
-        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 1
-        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 1
+        assert Path(tmp_path + "/bash-completion/completions/bdfr").exists() == 1
+        assert Path(tmp_path + "/fish/vendor_completions.d/bdfr.fish").exists() == 1
+        assert Path(tmp_path + "/zsh/site-functions/_bdfr").exists() == 1
         Completion("all").uninstall()
-        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 0
-        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 0
-        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 0
+        assert Path(tmp_path + "/bash-completion/completions/bdfr").exists() == 0
+        assert Path(tmp_path + "/fish/vendor_completions.d/bdfr.fish").exists() == 0
+        assert Path(tmp_path + "/zsh/site-functions/_bdfr").exists() == 0
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
 def test_cli_completion_bash(tmp_path: Path):
-    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+    tmp_path = str(tmp_path)
+    with patch("appdirs.user_data_dir", return_value=tmp_path):
         Completion("bash").install()
-        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 1
+        assert Path(tmp_path + "/bash-completion/completions/bdfr").exists() == 1
         Completion("bash").uninstall()
-        assert Path.exists(Path(str(tmp_path) + "/bash-completion/completions/bdfr")) == 0
+        assert Path(tmp_path + "/bash-completion/completions/bdfr").exists() == 0
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
 def test_cli_completion_fish(tmp_path: Path):
-    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+    tmp_path = str(tmp_path)
+    with patch("appdirs.user_data_dir", return_value=tmp_path):
         Completion("fish").install()
-        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 1
+        assert Path(tmp_path + "/fish/vendor_completions.d/bdfr.fish").exists() == 1
         Completion("fish").uninstall()
-        assert Path.exists(Path(str(tmp_path) + "/fish/vendor_completions.d/bdfr.fish")) == 0
+        assert Path(tmp_path + "/fish/vendor_completions.d/bdfr.fish").exists() == 0
 
 
 @pytest.mark.skipif(sys.platform == "win32", reason="Completions are not currently supported on Windows.")
 def test_cli_completion_zsh(tmp_path: Path):
-    with patch("appdirs.user_data_dir", return_value=str(tmp_path)):
+    tmp_path = str(tmp_path)
+    with patch("appdirs.user_data_dir", return_value=tmp_path):
         Completion("zsh").install()
-        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 1
+        assert Path(tmp_path + "/zsh/site-functions/_bdfr").exists() == 1
         Completion("zsh").uninstall()
-        assert Path.exists(Path(str(tmp_path) + "/zsh/site-functions/_bdfr")) == 0
+        assert Path(tmp_path + "/zsh/site-functions/_bdfr").exists() == 0

From 7fef403757203af6116fa37ccdcdac273df2018b Mon Sep 17 00:00:00 2001
From: OMEGA_RAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Tue, 20 Dec 2022 13:32:43 -0500
Subject: [PATCH 35/66] Update completion.py

---
 bdfr/completion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdfr/completion.py b/bdfr/completion.py
index 668fb2a..7b38322 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# coding=utf-8
+# -*- coding: utf-8 -*-
 
 import subprocess
 from os import environ

From fe9cc7f29fe81806e342141390352fe24cbd4da6 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Sat, 24 Dec 2022 20:52:45 -0500
Subject: [PATCH 36/66] Redgifs updates

Update Redgifs regex for further edge case.

Add test for checking ID.
---
 bdfr/site_downloaders/redgifs.py       |  9 +++++++--
 tests/site_downloaders/test_redgifs.py | 13 +++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py
index 40d1466..95d23d2 100644
--- a/bdfr/site_downloaders/redgifs.py
+++ b/bdfr/site_downloaders/redgifs.py
@@ -23,13 +23,18 @@ class Redgifs(BaseDownloader):
         return [Resource(self.post, m, Resource.retry_download(m), None) for m in media_urls]
 
     @staticmethod
-    def _get_link(url: str) -> set[str]:
+    def _get_id(url: str) -> str:
         try:
             if url.endswith("/"):
                 url = url.removesuffix("/")
-            redgif_id = re.match(r".*/(.*?)(\..{0,})?$", url).group(1)
+            redgif_id = re.match(r".*/(.*?)(?:\?.*|\..{0,})?$", url).group(1)
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Redgifs ID from {url}")
+        return redgif_id
+
+    @staticmethod
+    def _get_link(url: str) -> set[str]:
+        redgif_id = Redgifs._get_id(url)
 
         auth_token = json.loads(Redgifs.retrieve_url("https://api.redgifs.com/v2/auth/temporary").text)["token"]
         if not auth_token:
diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py
index 37e48e8..fd0e0ed 100644
--- a/tests/site_downloaders/test_redgifs.py
+++ b/tests/site_downloaders/test_redgifs.py
@@ -10,6 +10,19 @@ from bdfr.resource import Resource
 from bdfr.site_downloaders.redgifs import Redgifs
 
 
+@pytest.mark.parametrize(
+    ("test_url", "expected"),
+    (
+        ("https://redgifs.com/watch/frighteningvictorioussalamander", "frighteningvictorioussalamander"),
+        ("https://www.redgifs.com/watch/genuineprivateguillemot/", "genuineprivateguillemot"),
+        ("https://www.redgifs.com/watch/marriedcrushingcob?rel=u%3Akokiri.girl%3Bo%3Arecent", "marriedcrushingcob"),
+    ),
+)
+def test_get_id(test_url: str, expected: str):
+    result = Redgifs._get_id(test_url)
+    assert result == expected
+
+
 @pytest.mark.online
 @pytest.mark.parametrize(
     ("test_url", "expected"),

From 13887ca7e1acd605426537a6043bf901aee55515 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Mon, 26 Dec 2022 20:25:20 -0500
Subject: [PATCH 37/66] Redgif updates

Coverage for direct links.

The direct link won't work because it will have the wrong auth anyway but this will at least end up with the right API call.
---
 bdfr/site_downloaders/redgifs.py       | 4 +++-
 tests/site_downloaders/test_redgifs.py | 2 ++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py
index 95d23d2..205674a 100644
--- a/bdfr/site_downloaders/redgifs.py
+++ b/bdfr/site_downloaders/redgifs.py
@@ -27,7 +27,9 @@ class Redgifs(BaseDownloader):
         try:
             if url.endswith("/"):
                 url = url.removesuffix("/")
-            redgif_id = re.match(r".*/(.*?)(?:\?.*|\..{0,})?$", url).group(1)
+            redgif_id = re.match(r".*/(.*?)(?:\?.*|\..{0,})?$", url).group(1).lower()
+            if redgif_id.endswith("-mobile"):
+                redgif_id = redgif_id.removesuffix("-mobile")
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Redgifs ID from {url}")
         return redgif_id
diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py
index fd0e0ed..5f4f6fc 100644
--- a/tests/site_downloaders/test_redgifs.py
+++ b/tests/site_downloaders/test_redgifs.py
@@ -16,6 +16,8 @@ from bdfr.site_downloaders.redgifs import Redgifs
         ("https://redgifs.com/watch/frighteningvictorioussalamander", "frighteningvictorioussalamander"),
         ("https://www.redgifs.com/watch/genuineprivateguillemot/", "genuineprivateguillemot"),
         ("https://www.redgifs.com/watch/marriedcrushingcob?rel=u%3Akokiri.girl%3Bo%3Arecent", "marriedcrushingcob"),
+        ("https://thumbs4.redgifs.com/DismalIgnorantDrongo.mp4", "dismalignorantdrongo"),
+        ("https://thumbs4.redgifs.com/DismalIgnorantDrongo-mobile.mp4", "dismalignorantdrongo"),
     ),
 )
 def test_get_id(test_url: str, expected: str):

From 2bafb1b99b552e2817484adea21cab087077dda0 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Wed, 28 Dec 2022 10:00:43 -0500
Subject: [PATCH 38/66] Consolidate flake8 settings

Consolidates sane flake8 settings to pyproject with the Flake8-pyproject plugin.

Does not change logic of test workflow but allows base settings to live in pyproject for anyone using flake8 as an external linter (e.g. vscode)

Also fixes some flake8 errors that were not being picked up by current testing, mostly unused imports.
---
 .github/workflows/test.yml       | 8 ++++++--
 README.md                        | 8 ++++++--
 bdfr/connector.py                | 4 ++--
 bdfr/site_downloaders/imgur.py   | 2 +-
 bdfr/site_downloaders/vreddit.py | 7 ++-----
 pyproject.toml                   | 6 ++++++
 tests/test_download_filter.py    | 2 +-
 7 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0d52ef6..ca32bb3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -3,8 +3,12 @@ name: Python Test
 on:
   push:
     branches: [ master, development ]
+    paths-ignore:
+      - "*.md"
   pull_request:
     branches: [ master, development ]
+    paths-ignore:
+      - "*.md"
 
 jobs:
   test:
@@ -27,7 +31,7 @@ jobs:
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip flake8 pytest pytest-cov
+        python -m pip install --upgrade pip Flake8-pyproject pytest pytest-cov
         pip install .
 
     - name: Make configuration for tests
@@ -38,7 +42,7 @@ jobs:
 
     - name: Lint with flake8
       run: |
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --select=E9,F63,F7,F82
 
     - name: Test with pytest
       run: |
diff --git a/README.md b/README.md
index 120448e..dd700ad 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,12 @@
 # Bulk Downloader for Reddit
 
-[![PyPI version](https://img.shields.io/pypi/v/bdfr.svg)](https://pypi.python.org/pypi/bdfr)
-[![PyPI downloads](https://img.shields.io/pypi/dm/bdfr)](https://pypi.python.org/pypi/bdfr)
+[![PyPI Status](https://img.shields.io/pypi/status/bdfr?logo=PyPI)](https://pypi.python.org/pypi/bdfr)
+[![PyPI version](https://img.shields.io/pypi/v/bdfr.svg?logo=PyPI)](https://pypi.python.org/pypi/bdfr)
+[![PyPI downloads](https://img.shields.io/pypi/dm/bdfr?logo=PyPI)](https://pypi.python.org/pypi/bdfr)
+[![AUR version](https://img.shields.io/aur/version/python-bdfr?logo=Arch%20Linux)](https://aur.archlinux.org/packages/python-bdfr)
 [![Python Test](https://github.com/aliparlakci/bulk-downloader-for-reddit/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/aliparlakci/bulk-downloader-for-reddit/actions/workflows/test.yml)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?logo=Python)](https://github.com/psf/black)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
 
 This is a tool to download submissions or submission data from Reddit. It can be used to archive data or even crawl Reddit to gather research data. The BDFR is flexible and can be used in scripts if needed through an extensive command-line interface. [List of currently supported sources](#list-of-currently-supported-sources)
 
diff --git a/bdfr/connector.py b/bdfr/connector.py
index bf50f32..d0c4ac2 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -205,7 +205,7 @@ class RedditConnector(metaclass=ABCMeta):
         else:
             log_path = Path(self.args.log).resolve().expanduser()
             if not log_path.parent.exists():
-                raise errors.BulkDownloaderException(f"Designated location for logfile does not exist")
+                raise errors.BulkDownloaderException("Designated location for logfile does not exist")
         backup_count = self.cfg_parser.getint("DEFAULT", "backup_log_count", fallback=3)
         file_handler = logging.handlers.RotatingFileHandler(
             log_path,
@@ -323,7 +323,7 @@ class RedditConnector(metaclass=ABCMeta):
     def get_multireddits(self) -> list[Iterator]:
         if self.args.multireddit:
             if len(self.args.user) != 1:
-                logger.error(f"Only 1 user can be supplied when retrieving from multireddits")
+                logger.error("Only 1 user can be supplied when retrieving from multireddits")
                 return []
             out = []
             for multi in self.split_args_input(self.args.multireddit):
diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index a4c378f..537ab64 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -67,7 +67,7 @@ class Imgur(BaseDownloader):
             image_dict = re.search(outer_regex, chosen_script).group(1)
             image_dict = re.search(inner_regex, image_dict).group(1)
         except AttributeError:
-            raise SiteDownloaderError(f"Could not find image dictionary in page source")
+            raise SiteDownloaderError("Could not find image dictionary in page source")
 
         try:
             image_dict = json.loads(image_dict)
diff --git a/bdfr/site_downloaders/vreddit.py b/bdfr/site_downloaders/vreddit.py
index 8f6022e..48f5ba1 100644
--- a/bdfr/site_downloaders/vreddit.py
+++ b/bdfr/site_downloaders/vreddit.py
@@ -2,14 +2,11 @@
 # -*- coding: utf-8 -*-
 
 import logging
-import tempfile
-from pathlib import Path
-from typing import Callable, Optional
+from typing import Optional
 
-import yt_dlp
 from praw.models import Submission
 
-from bdfr.exceptions import NotADownloadableLinkError, SiteDownloaderError
+from bdfr.exceptions import NotADownloadableLinkError
 from bdfr.resource import Resource
 from bdfr.site_authenticator import SiteAuthenticator
 from bdfr.site_downloaders.youtube import Youtube
diff --git a/pyproject.toml b/pyproject.toml
index 6879a6d..a8b0dd2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -62,6 +62,12 @@ bdfr-download = "bdfr.__main__:cli_download"
 [tool.black]
 line-length = 120
 
+[tool.flake8]
+exclude = ["scripts"]
+max-line-length = 120
+show-source = true
+statistics = true
+
 [tool.isort]
 profile = "black"
 py_version = 39
diff --git a/tests/test_download_filter.py b/tests/test_download_filter.py
index 3b4d6b8..6062dc3 100644
--- a/tests/test_download_filter.py
+++ b/tests/test_download_filter.py
@@ -76,4 +76,4 @@ def test_filter_empty_filter(test_url: str):
     download_filter = DownloadFilter()
     test_resource = Resource(MagicMock(), test_url, lambda: None)
     result = download_filter.check_resource(test_resource)
-    assert result is True
+    assert result

From 874c7e3117450f5ca477ee80b048653b5360df35 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Sat, 31 Dec 2022 08:53:13 -0500
Subject: [PATCH 39/66] Redgif fixes

Missing half of #733
---
 bdfr/site_downloaders/download_factory.py       | 2 +-
 tests/site_downloaders/test_download_factory.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index df2a0ea..6237ecd 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -28,7 +28,7 @@ class DownloadFactory:
         sanitised_url = DownloadFactory.sanitise_url(url)
         if re.match(r"(i\.|m\.)?imgur", sanitised_url):
             return Imgur
-        elif re.match(r"(i\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
+        elif re.match(r"(i\.|thumbs\d\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
             return Redgifs
         elif re.match(r".*/.*\.[a-zA-Z34]{3,4}(\?[\w;&=]*)?$", sanitised_url) and not DownloadFactory.is_web_resource(
             sanitised_url
diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py
index bb09471..062635c 100644
--- a/tests/site_downloaders/test_download_factory.py
+++ b/tests/site_downloaders/test_download_factory.py
@@ -40,6 +40,7 @@ from bdfr.site_downloaders.youtube import Youtube
         ("https://youtube.com/watch?v=Gv8Wz74FjVA", Youtube),
         ("https://redgifs.com/watch/courageousimpeccablecanvasback", Redgifs),
         ("https://www.gifdeliverynetwork.com/repulsivefinishedandalusianhorse", Redgifs),
+        ("https://thumbs4.redgifs.com/DismalIgnorantDrongo-mobile.mp4", Redgifs),
         ("https://youtu.be/DevfjHOhuFc", Youtube),
         ("https://m.youtube.com/watch?v=kr-FeojxzUM", Youtube),
         ("https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781", Direct),

From c4bece2f5844e41e4771c7a95b857474ac0ab030 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 31 Dec 2022 09:09:48 -0500
Subject: [PATCH 40/66] Add flake8 to precommit

adds flake8 to pre-commit and dev requirements.
---
 .pre-commit-config.yaml | 6 ++++++
 docs/CONTRIBUTING.md    | 3 ++-
 pyproject.toml          | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7013228..add4ea6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,6 +13,12 @@ repos:
       - id: isort
         name: isort (python)
 
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        additional_dependencies: [Flake8-pyproject]
+
   - repo: https://github.com/markdownlint/markdownlint
     rev: v0.12.0
     hooks:
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 72666e7..53fcb03 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -69,12 +69,13 @@ python3 -m pip install -e .[dev]
 The BDFR project uses several tools to manage the code of the project. These include:
 
 - [black](https://github.com/psf/black)
+- [flake8](https://github.com/john-hen/Flake8-pyproject)
 - [isort](https://github.com/PyCQA/isort)
 - [markdownlint (mdl)](https://github.com/markdownlint/markdownlint)
 - [tox](https://tox.wiki/en/latest/)
 - [pre-commit](https://github.com/pre-commit/pre-commit)
 
-The first three tools are formatters. These change the code to the standards expected for the BDFR project. The configuration details for these tools are contained in the [pyproject.toml](../pyproject.toml) file for the project.
+The first four tools are formatters. These change the code to the standards expected for the BDFR project. The configuration details for these tools are contained in the [pyproject.toml](../pyproject.toml) file for the project.
 
 The tool `tox` is used to run tests and tools on demand and has the following environments:
 
diff --git a/pyproject.toml b/pyproject.toml
index a8b0dd2..4a28a8f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,6 +42,7 @@ data-files = {"config" = ["bdfr/default_config.cfg",]}
 [project.optional-dependencies]
 dev = [
     "black>=22.10.0",
+    "Flake8-pyproject>=1.2.2",
     "isort>=5.10.1",
     "pre-commit>=2.20.0",
     "pytest>=7.1.0",

From b6edc367532a42cd91161450c8deb829e4d3780c Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sun, 1 Jan 2023 03:46:08 -0500
Subject: [PATCH 41/66] Update connector for 7 digit ID's

---
 bdfr/connector.py       | 2 +-
 tests/test_connector.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdfr/connector.py b/bdfr/connector.py
index bf50f32..1583e37 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -301,7 +301,7 @@ class RedditConnector(metaclass=ABCMeta):
     def get_submissions_from_link(self) -> list[list[praw.models.Submission]]:
         supplied_submissions = []
         for sub_id in self.args.link:
-            if len(sub_id) == 6:
+            if len(sub_id) in (6, 7):
                 supplied_submissions.append(self.reddit_instance.submission(id=sub_id))
             else:
                 supplied_submissions.append(self.reddit_instance.submission(url=sub_id))
diff --git a/tests/test_connector.py b/tests/test_connector.py
index 9681e4b..bf781e2 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -167,6 +167,7 @@ def test_create_authenticator(downloader_mock: MagicMock):
         ("lvpf4l",),
         ("lvpf4l", "lvqnsn"),
         ("lvpf4l", "lvqnsn", "lvl9kd"),
+        ("1000000",),
     ),
 )
 def test_get_submissions_from_link(

From 3fdaf353068b026468186ad8f57ac403c4767760 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sun, 1 Jan 2023 08:59:34 -0500
Subject: [PATCH 42/66] Update black/isort

---
 .pre-commit-config.yaml | 4 ++--
 pyproject.toml          | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index add4ea6..28bd140 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,12 +3,12 @@
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.10.0
+    rev: 22.12.0
     hooks:
       - id: black
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.11.4
     hooks:
       - id: isort
         name: isort (python)
diff --git a/pyproject.toml b/pyproject.toml
index 4a28a8f..c88008d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,9 +41,9 @@ data-files = {"config" = ["bdfr/default_config.cfg",]}
 
 [project.optional-dependencies]
 dev = [
-    "black>=22.10.0",
+    "black>=22.12.0",
     "Flake8-pyproject>=1.2.2",
-    "isort>=5.10.1",
+    "isort>=5.11.4",
     "pre-commit>=2.20.0",
     "pytest>=7.1.0",
     "tox>=3.27.1",

From aced16456027f48427cd348fe8eb3bd8ba02d199 Mon Sep 17 00:00:00 2001
From: OMEGA_RAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sun, 1 Jan 2023 23:11:43 -0500
Subject: [PATCH 43/66] Update test.yml

Corrects .md ignore and adds markdown lint files as there would be no need to trigger python tests from them.
---
 .github/workflows/test.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ca32bb3..927d70a 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -4,11 +4,15 @@ on:
   push:
     branches: [ master, development ]
     paths-ignore:
-      - "*.md"
+      - "**.md"
+      - ".markdown_style.rb"
+      - ".mdlrc"
   pull_request:
     branches: [ master, development ]
     paths-ignore:
-      - "*.md"
+      - "**.md"
+      - ".markdown_style.rb"
+      - ".mdlrc"
 
 jobs:
   test:

From e0e780a272a27ba22149af54664b4e214517cbb0 Mon Sep 17 00:00:00 2001
From: Brian <brianbotterill@me.com>
Date: Mon, 2 Jan 2023 16:17:52 -0700
Subject: [PATCH 44/66] Update README.md to include information from #549

Serene-Arc determined that the time filter does not apply if the reddit API is not providing the "Top" or "Controversial" list.

As such, I have updated the time option documentation to clarify that.
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index dd700ad..a174d80 100644
--- a/README.md
+++ b/README.md
@@ -197,6 +197,7 @@ The following options are common between both the `archive` and `download` comma
 - `-t, --time`
     - This is the time filter that will be applied to all applicable sources
     - This option does not apply to upvoted or saved posts when scraping from these sources
+    - This option only applies if sorting by top or controversial.  See --sort for more detail.
     - The following options are available:
         - `all` (default)
         - `hour`

From f40ac35f4af9f26223cc12eb78a1f10c8ff83a1f Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Tue, 3 Jan 2023 19:16:23 +1000
Subject: [PATCH 45/66] Add option to determine restriction scheme

---
 bdfr/file_name_formatter.py       | 15 ++++++++++++---
 tests/test_file_name_formatter.py | 24 ++++++++++++++++++++++--
 2 files changed, 34 insertions(+), 5 deletions(-)

diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py
index 9ee481d..0330336 100644
--- a/bdfr/file_name_formatter.py
+++ b/bdfr/file_name_formatter.py
@@ -28,12 +28,19 @@ class FileNameFormatter:
         "upvotes",
     )
 
-    def __init__(self, file_format_string: str, directory_format_string: str, time_format_string: str):
+    def __init__(
+        self,
+        file_format_string: str,
+        directory_format_string: str,
+        time_format_string: str,
+        restriction_scheme: Optional[str] = None,
+    ):
         if not self.validate_string(file_format_string):
             raise BulkDownloaderException(f'"{file_format_string}" is not a valid format string')
         self.file_format_string = file_format_string
         self.directory_format_string: list[str] = directory_format_string.split("/")
         self.time_format_string = time_format_string
+        self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None
 
     def _format_name(self, submission: Union[Comment, Submission], format_string: str) -> str:
         if isinstance(submission, Submission):
@@ -52,9 +59,11 @@ class FileNameFormatter:
 
         result = result.replace("/", "")
 
-        if platform.system() == "Windows":
+        if self.restiction_scheme is None:
+            if platform.system() == "Windows":
+                result = FileNameFormatter._format_for_windows(result)
+        elif self.restiction_scheme == "windows":
             result = FileNameFormatter._format_for_windows(result)
-
         return result
 
     @staticmethod
diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py
index 4964b3b..5f94e5f 100644
--- a/tests/test_file_name_formatter.py
+++ b/tests/test_file_name_formatter.py
@@ -33,6 +33,10 @@ def submission() -> MagicMock:
     return test
 
 
+def check_valid_windows_path(test_string: str):
+    return test_string == FileNameFormatter._format_for_windows(test_string)
+
+
 def do_test_string_equality(result: Union[Path, str], expected: str) -> bool:
     if platform.system() == "Windows":
         expected = FileNameFormatter._format_for_windows(expected)
@@ -91,6 +95,15 @@ def test_check_format_string_validity(test_string: str, expected: bool):
 
 @pytest.mark.online
 @pytest.mark.reddit
+@pytest.mark.parametrize(
+    "restriction_scheme",
+    (
+        "windows",
+        "linux",
+        "bla",
+        None,
+    ),
+)
 @pytest.mark.parametrize(
     ("test_format_string", "expected"),
     (
@@ -102,10 +115,17 @@ def test_check_format_string_validity(test_string: str, expected: bool):
         ("{REDDITOR}_{TITLE}_{POSTID}", "Kirsty-Blue_George Russel acknowledges the Twitter trend about him_w22m5l"),
     ),
 )
-def test_format_name_real(test_format_string: str, expected: str, reddit_submission: praw.models.Submission):
-    test_formatter = FileNameFormatter(test_format_string, "", "")
+def test_format_name_real(
+    test_format_string: str,
+    expected: str,
+    reddit_submission: praw.models.Submission,
+    restriction_scheme: Optional[str],
+):
+    test_formatter = FileNameFormatter(test_format_string, "", "", restriction_scheme)
     result = test_formatter._format_name(reddit_submission, test_format_string)
     assert do_test_string_equality(result, expected)
+    if restriction_scheme == "windows":
+        assert check_valid_windows_path(result)
 
 
 @pytest.mark.online

From 4f07e92c5ee47d38941815d2fe48d90de884e287 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Wed, 4 Jan 2023 19:04:31 +1000
Subject: [PATCH 46/66] Add option to classes

---
 bdfr/__main__.py      | 3 ++-
 bdfr/configuration.py | 1 +
 bdfr/connector.py     | 4 ++++
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/bdfr/__main__.py b/bdfr/__main__.py
index e35ba0a..2823ce1 100644
--- a/bdfr/__main__.py
+++ b/bdfr/__main__.py
@@ -20,15 +20,16 @@ _common_options = [
     click.argument("directory", type=str),
     click.option("--authenticate", is_flag=True, default=None),
     click.option("--config", type=str, default=None),
-    click.option("--opts", type=str, default=None),
     click.option("--disable-module", multiple=True, default=None, type=str),
     click.option("--exclude-id", default=None, multiple=True),
     click.option("--exclude-id-file", default=None, multiple=True),
     click.option("--file-scheme", default=None, type=str),
+    click.option("--filename-restriction-scheme", type=click.Choice(("linux", "windows")), default=None),
     click.option("--folder-scheme", default=None, type=str),
     click.option("--ignore-user", type=str, multiple=True, default=None),
     click.option("--include-id-file", multiple=True, default=None),
     click.option("--log", type=str, default=None),
+    click.option("--opts", type=str, default=None),
     click.option("--saved", is_flag=True, default=None),
     click.option("--search", default=None, type=str),
     click.option("--submitted", is_flag=True, default=None),
diff --git a/bdfr/configuration.py b/bdfr/configuration.py
index 0d00192..78ae12e 100644
--- a/bdfr/configuration.py
+++ b/bdfr/configuration.py
@@ -23,6 +23,7 @@ class Configuration(Namespace):
         self.exclude_id = []
         self.exclude_id_file = []
         self.file_scheme: str = "{REDDITOR}_{TITLE}_{POSTID}"
+        self.filename_restriction_scheme = None
         self.folder_scheme: str = "{SUBREDDIT}"
         self.ignore_user = []
         self.include_id_file = []
diff --git a/bdfr/connector.py b/bdfr/connector.py
index 6d7bc64..8fe149a 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -107,6 +107,10 @@ class RedditConnector(metaclass=ABCMeta):
             self.args.time_format = option
         if not self.args.disable_module:
             self.args.disable_module = [self.cfg_parser.get("DEFAULT", "disabled_modules", fallback="")]
+        if not self.args.filename_restriction_scheme:
+            self.args.filename_restriction_scheme = self.cfg_parser.get(
+                "DEFAULT", "filename_restriction_scheme", fallback=None
+            )
         # Update config on disk
         with open(self.config_location, "w") as file:
             self.cfg_parser.write(file)

From 8c57dc228370d101722ec9d9e23786305c7e8638 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Wed, 4 Jan 2023 19:07:31 +1000
Subject: [PATCH 47/66] Add missing pytest flags to test

---
 tests/integration_tests/test_download_integration.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 138ea61..545bd31 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -400,6 +400,8 @@ def test_cli_download_score_filter(test_args: list[str], was_filtered: bool, tmp
     assert ("filtered due to score" in result.output) == was_filtered
 
 
+@pytest.mark.online
+@pytest.mark.reddit
 @pytest.mark.skipif(not does_test_config_exist, reason="A test config file is required for integration tests")
 @pytest.mark.parametrize(
     ("test_args", "response"),

From b64f5080257711465e2c752cda8c56d3bacc8e51 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Wed, 4 Jan 2023 19:37:02 +1000
Subject: [PATCH 48/66] Conform path length to filename scheme restriction

---
 bdfr/file_name_formatter.py       | 16 +++++++++++-----
 tests/test_file_name_formatter.py | 24 +++++++++++++++++-------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py
index 0330336..dd04fad 100644
--- a/bdfr/file_name_formatter.py
+++ b/bdfr/file_name_formatter.py
@@ -27,6 +27,8 @@ class FileNameFormatter:
         "title",
         "upvotes",
     )
+    WINDOWS_MAX_PATH_LENGTH = 260
+    LINUX_MAX_PATH_LENGTH = 4096
 
     def __init__(
         self,
@@ -41,6 +43,10 @@ class FileNameFormatter:
         self.directory_format_string: list[str] = directory_format_string.split("/")
         self.time_format_string = time_format_string
         self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None
+        if self.restiction_scheme == "windows":
+            self.max_path = self.WINDOWS_MAX_PATH_LENGTH
+        else:
+            self.max_path = self.find_max_path_length()
 
     def _format_name(self, submission: Union[Comment, Submission], format_string: str) -> str:
         if isinstance(submission, Submission):
@@ -63,6 +69,7 @@ class FileNameFormatter:
             if platform.system() == "Windows":
                 result = FileNameFormatter._format_for_windows(result)
         elif self.restiction_scheme == "windows":
+            logger.debug("Forcing Windows-compatible filenames")
             result = FileNameFormatter._format_for_windows(result)
         return result
 
@@ -135,14 +142,13 @@ class FileNameFormatter:
             raise BulkDownloaderException(f"Could not determine path name: {subfolder}, {index}, {resource.extension}")
         return file_path
 
-    @staticmethod
-    def limit_file_name_length(filename: str, ending: str, root: Path) -> Path:
+    def limit_file_name_length(self, filename: str, ending: str, root: Path) -> Path:
         root = root.resolve().expanduser()
         possible_id = re.search(r"((?:_\w{6})?$)", filename)
         if possible_id:
             ending = possible_id.group(1) + ending
             filename = filename[: possible_id.start()]
-        max_path = FileNameFormatter.find_max_path_length()
+        max_path = self.max_path
         max_file_part_length_chars = 255 - len(ending)
         max_file_part_length_bytes = 255 - len(ending.encode("utf-8"))
         max_path_length = max_path - len(ending) - len(str(root)) - 1
@@ -166,9 +172,9 @@ class FileNameFormatter:
             return int(subprocess.check_output(["getconf", "PATH_MAX", "/"]))
         except (ValueError, subprocess.CalledProcessError, OSError):
             if platform.system() == "Windows":
-                return 260
+                return FileNameFormatter.WINDOWS_MAX_PATH_LENGTH
             else:
-                return 4096
+                return FileNameFormatter.LINUX_MAX_PATH_LENGTH
 
     def format_resource_paths(
         self,
diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py
index 5f94e5f..fb34a53 100644
--- a/tests/test_file_name_formatter.py
+++ b/tests/test_file_name_formatter.py
@@ -33,6 +33,12 @@ def submission() -> MagicMock:
     return test
 
 
+@pytest.fixture()
+def test_formatter() -> FileNameFormatter:
+    out = FileNameFormatter("{TITLE}", "", "ISO")
+    return out
+
+
 def check_valid_windows_path(test_string: str):
     return test_string == FileNameFormatter._format_for_windows(test_string)
 
@@ -231,8 +237,8 @@ def test_format_multiple_resources():
         ("😍💕✨" * 100, "_1.png"),
     ),
 )
-def test_limit_filename_length(test_filename: str, test_ending: str):
-    result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path("."))
+def test_limit_filename_length(test_filename: str, test_ending: str, test_formatter: FileNameFormatter):
+    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
     assert len(result.name) <= 255
     assert len(result.name.encode("utf-8")) <= 255
     assert len(str(result)) <= FileNameFormatter.find_max_path_length()
@@ -253,8 +259,10 @@ def test_limit_filename_length(test_filename: str, test_ending: str):
         ("😍💕✨" * 100 + "_aaa1aa", "_1.png", "_aaa1aa_1.png"),
     ),
 )
-def test_preserve_id_append_when_shortening(test_filename: str, test_ending: str, expected_end: str):
-    result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path("."))
+def test_preserve_id_append_when_shortening(
+    test_filename: str, test_ending: str, expected_end: str, test_formatter: FileNameFormatter
+):
+    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
     assert len(result.name) <= 255
     assert len(result.name.encode("utf-8")) <= 255
     assert result.name.endswith(expected_end)
@@ -284,8 +292,8 @@ def test_shorten_filename_real(submission: MagicMock, tmp_path: Path):
         ("a" * 500, "_bbbbbb.jpg"),
     ),
 )
-def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path):
-    result = FileNameFormatter.limit_file_name_length(test_name, test_ending, tmp_path)
+def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path, test_formatter: FileNameFormatter):
+    result = test_formatter.limit_file_name_length(test_name, test_ending, tmp_path)
     assert len(str(result.name)) <= 255
     assert len(str(result.name).encode("UTF-8")) <= 255
     assert len(str(result.name).encode("cp1252")) <= 255
@@ -482,7 +490,9 @@ def test_get_max_path_length():
 def test_windows_max_path(tmp_path: Path):
     with unittest.mock.patch("platform.system", return_value="Windows"):
         with unittest.mock.patch("bdfr.file_name_formatter.FileNameFormatter.find_max_path_length", return_value=260):
-            result = FileNameFormatter.limit_file_name_length("test" * 100, "_1.png", tmp_path)
+            mock = MagicMock()
+            mock.max_path = 260
+            result = FileNameFormatter.limit_file_name_length(mock, "test" * 100, "_1.png", tmp_path)
             assert len(str(result)) <= 260
             assert len(result.name) <= (260 - len(str(tmp_path)))
 

From 77a01e1627e8c47a6cf27f76894c08f661894d14 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Wed, 4 Jan 2023 19:49:09 +1000
Subject: [PATCH 49/66] Add tests for new option

---
 bdfr/connector.py                             |  5 ++++-
 .../test_download_integration.py              | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/bdfr/connector.py b/bdfr/connector.py
index 8fe149a..860750d 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -111,6 +111,7 @@ class RedditConnector(metaclass=ABCMeta):
             self.args.filename_restriction_scheme = self.cfg_parser.get(
                 "DEFAULT", "filename_restriction_scheme", fallback=None
             )
+            logger.debug(f"Setting filename restriction scheme to '{self.args.filename_restriction_scheme}'")
         # Update config on disk
         with open(self.config_location, "w") as file:
             self.cfg_parser.write(file)
@@ -399,7 +400,9 @@ class RedditConnector(metaclass=ABCMeta):
                 raise errors.BulkDownloaderException(f"User {name} is banned")
 
     def create_file_name_formatter(self) -> FileNameFormatter:
-        return FileNameFormatter(self.args.file_scheme, self.args.folder_scheme, self.args.time_format)
+        return FileNameFormatter(
+            self.args.file_scheme, self.args.folder_scheme, self.args.time_format, self.args.filename_restriction_scheme
+        )
 
     def create_time_filter(self) -> RedditTypes.TimeType:
         try:
diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 545bd31..35ca0c0 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -421,3 +421,22 @@ def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
             result = runner.invoke(cli, test_args)
             assert result.exit_code == 0
             assert f"received {response} HTTP response" in result.output
+
+
+@pytest.mark.online
+@pytest.mark.reddit
+@pytest.mark.skipif(not does_test_config_exist, reason="A test config file is required for integration tests")
+@pytest.mark.parametrize(
+    "test_args",
+    (
+        ["-l", "102vd5i", "--filename-restriction-scheme", "windows"],
+        ["-l", "m3hxzd", "--filename-restriction-scheme", "windows"],
+    ),
+)
+def test_cli_download_explicit_filename_restriction_scheme(test_args: list[str], tmp_path: Path):
+    runner = CliRunner()
+    test_args = create_basic_args_for_download_runner(test_args, tmp_path)
+    result = runner.invoke(cli, test_args)
+    assert result.exit_code == 0
+    assert "Downloaded submission" in result.output
+    assert "Forcing Windows-compatible filenames" in result.output

From 12311029e4d8f4354c14fbc2fd368e8fe3c34505 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Wed, 4 Jan 2023 19:49:50 +1000
Subject: [PATCH 50/66] Conform test name

---
 tests/integration_tests/test_download_integration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 35ca0c0..4dae353 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -410,7 +410,7 @@ def test_cli_download_score_filter(test_args: list[str], was_filtered: bool, tmp
         (["--user", "nasa", "--submitted"], 504),
     ),
 )
-def test_user_serv_fail(test_args: list[str], response: int, tmp_path: Path):
+def test_cli_download_user_reddit_server_error(test_args: list[str], response: int, tmp_path: Path):
     runner = CliRunner()
     test_args = create_basic_args_for_download_runner(test_args, tmp_path)
     with patch("bdfr.connector.sleep", return_value=None):

From 241021fa391a2e287d1d04ab86e7b4aa5b548741 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Thu, 5 Jan 2023 17:38:37 +1000
Subject: [PATCH 51/66] Add new option details

---
 README.md | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/README.md b/README.md
index a174d80..b0bce0a 100644
--- a/README.md
+++ b/README.md
@@ -143,6 +143,10 @@ The following options are common between both the `archive` and `download` comma
     - Can be specified multiple times
     - Disables certain modules from being used
     - See [Disabling Modules](#disabling-modules) for more information and a list of module names
+- `--filename-restriction-scheme`
+    - Can be: `windows`, `linux`
+    - Turns off the OS detection and specifies which system to use when making filenames
+    - See [Filesystem Restrictions](#filesystem-restrictions)
 - `--ignore-user`
     - This will add a user to ignore
     - Can be specified multiple times
@@ -372,6 +376,7 @@ The following keys are optional, and defaults will be used if they cannot be fou
 - `max_wait_time`
 - `time_format`
 - `disabled_modules`
+- `filename-restriction-scheme`
 
 All of these should not be modified unless you know what you're doing, as the default values will enable the BDFR to function just fine. A configuration is included in the BDFR when it is installed, and this will be placed in the configuration directory as the default.
 
@@ -421,6 +426,14 @@ Running scenarios concurrently (at the same time) however, is more complicated.
 
 The way to fix this is to use the `--log` option to manually specify where the logfile is to be stored. If the given location is unique to each instance of the BDFR, then it will run fine.
 
+## Filesystem Restrictions
+
+Different filesystems have different restrictions for what files and directories can be named. Thesse are separated into two broad categories: Linux-based filesystems, which have very few restrictions; and Windows-based filesystems, which are much more restrictive in terms if forbidden characters and length of paths.
+
+During the normal course of operation, the BDFR detects what filesystem it is running on and formats any filenames and directories to conform to the rules that are expected of it. However, there are cases where this will fail. When running on a Linux-based machine, or another system where the home filesystem is permissive, and accessing a share or drive with a less permissive system, the BDFR will assume that the *home* filesystem's rules apply. For example, when downloading to a SAMBA share from Ubuntu, there will be errors as SAMBA is more restrictive than Ubuntu.
+
+The best option would be to always download to a filesystem that is as permission as possible, such as an NFS share or ext4 drive. However, when this is not possible, the BDFR allows for the restriction scheme to be manually specified at either the command-line or in the configuration file. At the command-line, this is done with `--filename-restriction-scheme windows`, or else an option by the same name in the configuration file.
+
 ## Manipulating Logfiles
 
 The logfiles that the BDFR outputs are consistent and quite detailed and in a format that is amenable to regex. To this end, a number of bash scripts have been [included here](./scripts). They show examples for how to extract successfully downloaded IDs, failed IDs, and more besides.

From 37dd7f11a8a8b06ce2bba801aa0245cddb818ff5 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Thu, 5 Jan 2023 20:01:00 +1000
Subject: [PATCH 52/66] Add some wiggle room to test results

---
 tests/test_connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_connector.py b/tests/test_connector.py
index bf781e2..c05988c 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -254,7 +254,7 @@ def test_get_subreddit_time_verification(
     for r in results:
         result_time = datetime.fromtimestamp(r.created_utc)
         time_diff = nowtime - result_time
-        assert time_diff < test_delta
+        assert abs(time_diff - test_delta) < timedelta(seconds=1)
 
 
 @pytest.mark.online

From 468b5a33aeaa38c5266134a26176ae1525d4438e Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Thu, 5 Jan 2023 20:01:49 +1000
Subject: [PATCH 53/66] Switch to a minute wiggle room

---
 tests/test_connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_connector.py b/tests/test_connector.py
index c05988c..41c44e3 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -254,7 +254,7 @@ def test_get_subreddit_time_verification(
     for r in results:
         result_time = datetime.fromtimestamp(r.created_utc)
         time_diff = nowtime - result_time
-        assert abs(time_diff - test_delta) < timedelta(seconds=1)
+        assert abs(time_diff - test_delta) < timedelta(minutes=1)
 
 
 @pytest.mark.online

From 579c5ab8eb4916e7bad8278851707b5614851f4c Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Fri, 6 Jan 2023 11:56:54 -0500
Subject: [PATCH 54/66] Add new Redgifs subdomain

Seems there's a v3 subdomain now (looks like it's mostly for mobile)
---
 bdfr/site_downloaders/download_factory.py       | 2 +-
 tests/site_downloaders/test_download_factory.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index 6237ecd..719d564 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -28,7 +28,7 @@ class DownloadFactory:
         sanitised_url = DownloadFactory.sanitise_url(url)
         if re.match(r"(i\.|m\.)?imgur", sanitised_url):
             return Imgur
-        elif re.match(r"(i\.|thumbs\d\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
+        elif re.match(r"(i\.|thumbs\d\.|v\d\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
             return Redgifs
         elif re.match(r".*/.*\.[a-zA-Z34]{3,4}(\?[\w;&=]*)?$", sanitised_url) and not DownloadFactory.is_web_resource(
             sanitised_url
diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py
index 062635c..f95e609 100644
--- a/tests/site_downloaders/test_download_factory.py
+++ b/tests/site_downloaders/test_download_factory.py
@@ -41,6 +41,7 @@ from bdfr.site_downloaders.youtube import Youtube
         ("https://redgifs.com/watch/courageousimpeccablecanvasback", Redgifs),
         ("https://www.gifdeliverynetwork.com/repulsivefinishedandalusianhorse", Redgifs),
         ("https://thumbs4.redgifs.com/DismalIgnorantDrongo-mobile.mp4", Redgifs),
+        ("https://v3.redgifs.com/watch/kaleidoscopicdaringvenomoussnake", Redgifs),
         ("https://youtu.be/DevfjHOhuFc", Youtube),
         ("https://m.youtube.com/watch?v=kr-FeojxzUM", Youtube),
         ("https://dynasty-scans.com/system/images_images/000/017/819/original/80215103_p0.png?1612232781", Direct),

From 59c25d21df63c92c7dd42fe23e809b3c6a562a03 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Fri, 6 Jan 2023 23:20:10 -0500
Subject: [PATCH 55/66] Update README.md

Updated the info on make-hard-links, no-dupes and search-existing to be more accurate to current functionality.
---
 README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3a8cef6..5a927bf 100644
--- a/README.md
+++ b/README.md
@@ -226,17 +226,18 @@ The following options are common between both the `archive` and `download` comma
 The following options apply only to the `download` command. This command downloads the files and resources linked to in the submission, or a text submission itself, to the disk in the specified directory.
 
 - `--make-hard-links`
-    - This flag will create hard links to an existing file when a duplicate is downloaded
+    - This flag will create hard links to an existing file when a duplicate is downloaded in the current run
     - This will make the file appear in multiple directories while only taking the space of a single instance
 - `--max-wait-time`
     - This option specifies the maximum wait time for downloading a resource
     - The default is 120 seconds
     - See [Rate Limiting](#rate-limiting) for details
 - `--no-dupes`
-    - This flag will not redownload files if they already exist somewhere in the root folder tree
+    - This flag will not redownload files if they were already downloaded in the current run
     - This is calculated by MD5 hash
 - `--search-existing`
-    - This will make the BDFR compile the hashes for every file in `directory` and store them to remove duplicates if `--no-dupes` is also supplied
+    - This will make the BDFR compile the hashes for every file in `directory`
+    - The hashes are used to remove duplicates if `--no-dupes` is supplied or make hard links if `--make-hard-links` is supplied
 - `--file-scheme`
     - Sets the scheme for files
     - Default is `{REDDITOR}_{TITLE}_{POSTID}`

From d18c30b481899eb9c24a76e10b54398b29d1dac0 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 7 Jan 2023 17:21:25 +1000
Subject: [PATCH 56/66] Fix failing tests

---
 tests/integration_tests/test_download_integration.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration_tests/test_download_integration.py b/tests/integration_tests/test_download_integration.py
index 4dae353..5d7238a 100644
--- a/tests/integration_tests/test_download_integration.py
+++ b/tests/integration_tests/test_download_integration.py
@@ -52,9 +52,9 @@ def create_basic_args_for_download_runner(test_args: list[str], run_path: Path):
         ["-s", "trollxchromosomes", "-L", 3, "--sort", "new"],
         ["-s", "trollxchromosomes", "-L", 3, "--time", "day", "--sort", "new"],
         ["-s", "trollxchromosomes", "-L", 3, "--search", "women"],
-        ["-s", "trollxchromosomes", "-L", 3, "--time", "day", "--search", "women"],
+        ["-s", "trollxchromosomes", "-L", 3, "--time", "week", "--search", "women"],
         ["-s", "trollxchromosomes", "-L", 3, "--sort", "new", "--search", "women"],
-        ["-s", "trollxchromosomes", "-L", 3, "--time", "day", "--sort", "new", "--search", "women"],
+        ["-s", "trollxchromosomes", "-L", 3, "--time", "week", "--sort", "new", "--search", "women"],
     ),
 )
 def test_cli_download_subreddits(test_args: list[str], tmp_path: Path):

From 9b23082a783c2f02c0f246774d79386187774ce9 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sat, 7 Jan 2023 17:21:54 +1000
Subject: [PATCH 57/66] Fix test

---
 tests/test_connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_connector.py b/tests/test_connector.py
index 41c44e3..d300b45 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -254,7 +254,7 @@ def test_get_subreddit_time_verification(
     for r in results:
         result_time = datetime.fromtimestamp(r.created_utc)
         time_diff = nowtime - result_time
-        assert abs(time_diff - test_delta) < timedelta(minutes=1)
+        assert time_diff < (test_delta + timedelta(minutes=1))
 
 
 @pytest.mark.online

From 8f17bcf43bc768763ea6ac89f3dd7471b9144c37 Mon Sep 17 00:00:00 2001
From: Serene-Arc <serenical@gmail.com>
Date: Sun, 8 Jan 2023 13:33:35 +1000
Subject: [PATCH 58/66] Modify code to accept pluggable logging handlers

---
 bdfr/__main__.py         | 22 ++++++++++++++--------
 bdfr/archiver.py         |  6 +++---
 bdfr/cloner.py           |  5 +++--
 bdfr/connector.py        | 26 +++++++++++++++-----------
 bdfr/downloader.py       |  5 +++--
 tests/test_downloader.py | 20 ++++++++++++--------
 6 files changed, 50 insertions(+), 34 deletions(-)

diff --git a/bdfr/__main__.py b/bdfr/__main__.py
index 2823ce1..dadba51 100644
--- a/bdfr/__main__.py
+++ b/bdfr/__main__.py
@@ -111,9 +111,10 @@ def cli_download(context: click.Context, **_):
     """Used to download content posted to Reddit."""
     config = Configuration()
     config.process_click_arguments(context)
-    setup_logging(config.verbose)
+    silence_module_loggers()
+    stream = make_console_logging_handler(config.verbose)
     try:
-        reddit_downloader = RedditDownloader(config)
+        reddit_downloader = RedditDownloader(config, [stream])
         reddit_downloader.download()
     except Exception:
         logger.exception("Downloader exited unexpectedly")
@@ -131,9 +132,10 @@ def cli_archive(context: click.Context, **_):
     """Used to archive post data from Reddit."""
     config = Configuration()
     config.process_click_arguments(context)
-    setup_logging(config.verbose)
+    silence_module_loggers()
+    stream = make_console_logging_handler(config.verbose)
     try:
-        reddit_archiver = Archiver(config)
+        reddit_archiver = Archiver(config, [stream])
         reddit_archiver.download()
     except Exception:
         logger.exception("Archiver exited unexpectedly")
@@ -152,9 +154,10 @@ def cli_clone(context: click.Context, **_):
     """Combines archive and download commands."""
     config = Configuration()
     config.process_click_arguments(context)
-    setup_logging(config.verbose)
+    silence_module_loggers()
+    stream = make_console_logging_handler(config.verbose)
     try:
-        reddit_scraper = RedditCloner(config)
+        reddit_scraper = RedditCloner(config, [stream])
         reddit_scraper.download()
     except Exception:
         logger.exception("Scraper exited unexpectedly")
@@ -187,7 +190,7 @@ def cli_completion(shell: str, uninstall: bool):
         Completion(shell).install()
 
 
-def setup_logging(verbosity: int):
+def make_console_logging_handler(verbosity: int) -> logging.StreamHandler:
     class StreamExceptionFilter(logging.Filter):
         def filter(self, record: logging.LogRecord) -> bool:
             result = not (record.levelno == logging.ERROR and record.exc_info)
@@ -200,13 +203,16 @@ def setup_logging(verbosity: int):
     formatter = logging.Formatter("[%(asctime)s - %(name)s - %(levelname)s] - %(message)s")
     stream.setFormatter(formatter)
 
-    logger.addHandler(stream)
     if verbosity <= 0:
         stream.setLevel(logging.INFO)
     elif verbosity == 1:
         stream.setLevel(logging.DEBUG)
     else:
         stream.setLevel(9)
+    return stream
+
+
+def silence_module_loggers():
     logging.getLogger("praw").setLevel(logging.CRITICAL)
     logging.getLogger("prawcore").setLevel(logging.CRITICAL)
     logging.getLogger("urllib3").setLevel(logging.CRITICAL)
diff --git a/bdfr/archiver.py b/bdfr/archiver.py
index be5a445..88136a8 100644
--- a/bdfr/archiver.py
+++ b/bdfr/archiver.py
@@ -5,7 +5,7 @@ import json
 import logging
 import re
 from time import sleep
-from typing import Iterator, Union
+from typing import Iterable, Iterator, Union
 
 import dict2xml
 import praw.models
@@ -24,8 +24,8 @@ logger = logging.getLogger(__name__)
 
 
 class Archiver(RedditConnector):
-    def __init__(self, args: Configuration):
-        super(Archiver, self).__init__(args)
+    def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
+        super(Archiver, self).__init__(args, logging_handlers)
 
     def download(self):
         for generator in self.reddit_lists:
diff --git a/bdfr/cloner.py b/bdfr/cloner.py
index 53108c0..c31f6cc 100644
--- a/bdfr/cloner.py
+++ b/bdfr/cloner.py
@@ -3,6 +3,7 @@
 
 import logging
 from time import sleep
+from typing import Iterable
 
 import prawcore
 
@@ -14,8 +15,8 @@ logger = logging.getLogger(__name__)
 
 
 class RedditCloner(RedditDownloader, Archiver):
-    def __init__(self, args: Configuration):
-        super(RedditCloner, self).__init__(args)
+    def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
+        super(RedditCloner, self).__init__(args, logging_handlers)
 
     def download(self):
         for generator in self.reddit_lists:
diff --git a/bdfr/connector.py b/bdfr/connector.py
index 860750d..89339f0 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -14,7 +14,7 @@ from datetime import datetime
 from enum import Enum, auto
 from pathlib import Path
 from time import sleep
-from typing import Callable, Iterator
+from typing import Callable, Iterable, Iterator
 
 import appdirs
 import praw
@@ -51,20 +51,20 @@ class RedditTypes:
 
 
 class RedditConnector(metaclass=ABCMeta):
-    def __init__(self, args: Configuration):
+    def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
         self.args = args
         self.config_directories = appdirs.AppDirs("bdfr", "BDFR")
+        self.determine_directories()
+        self.load_config()
+        self.read_config()
+        file_log = self.create_file_logger()
+        self._apply_logging_handlers(itertools.chain(logging_handlers, [file_log]))
         self.run_time = datetime.now().isoformat()
         self._setup_internal_objects()
 
         self.reddit_lists = self.retrieve_reddit_lists()
 
     def _setup_internal_objects(self):
-        self.determine_directories()
-        self.load_config()
-        self.create_file_logger()
-
-        self.read_config()
 
         self.parse_disabled_modules()
 
@@ -94,6 +94,12 @@ class RedditConnector(metaclass=ABCMeta):
         self.args.skip_subreddit = self.split_args_input(self.args.skip_subreddit)
         self.args.skip_subreddit = {sub.lower() for sub in self.args.skip_subreddit}
 
+    @staticmethod
+    def _apply_logging_handlers(handlers: Iterable[logging.Handler]):
+        main_logger = logging.getLogger()
+        for handler in handlers:
+            main_logger.addHandler(handler)
+
     def read_config(self):
         """Read any cfg values that need to be processed"""
         if self.args.max_wait_time is None:
@@ -203,8 +209,7 @@ class RedditConnector(metaclass=ABCMeta):
             raise errors.BulkDownloaderException("Could not find a configuration file to load")
         self.cfg_parser.read(self.config_location)
 
-    def create_file_logger(self):
-        main_logger = logging.getLogger()
+    def create_file_logger(self) -> logging.handlers.RotatingFileHandler:
         if self.args.log is None:
             log_path = Path(self.config_directory, "log_output.txt")
         else:
@@ -229,8 +234,7 @@ class RedditConnector(metaclass=ABCMeta):
         formatter = logging.Formatter("[%(asctime)s - %(name)s - %(levelname)s] - %(message)s")
         file_handler.setFormatter(formatter)
         file_handler.setLevel(0)
-
-        main_logger.addHandler(file_handler)
+        return file_handler
 
     @staticmethod
     def sanitise_subreddit_name(subreddit: str) -> str:
diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index 7ad8a6b..31c839d 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -9,6 +9,7 @@ from datetime import datetime
 from multiprocessing import Pool
 from pathlib import Path
 from time import sleep
+from typing import Iterable
 
 import praw
 import praw.exceptions
@@ -36,8 +37,8 @@ def _calc_hash(existing_file: Path):
 
 
 class RedditDownloader(RedditConnector):
-    def __init__(self, args: Configuration):
-        super(RedditDownloader, self).__init__(args)
+    def __init__(self, args: Configuration, logging_handlers: Iterable[logging.Handler] = ()):
+        super(RedditDownloader, self).__init__(args, logging_handlers)
         if self.args.search_existing:
             self.master_hash_list = self.scan_existing_files(self.download_directory)
 
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index d7aa8dd..b78a81c 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-
+import logging
 import os
 import re
 from pathlib import Path
@@ -9,12 +9,16 @@ from unittest.mock import MagicMock, patch
 import praw.models
 import pytest
 
-from bdfr.__main__ import setup_logging
+from bdfr.__main__ import make_console_logging_handler
 from bdfr.configuration import Configuration
 from bdfr.connector import RedditConnector
 from bdfr.downloader import RedditDownloader
 
 
+def add_console_handler():
+    logging.getLogger().addHandler(make_console_logging_handler(3))
+
+
 @pytest.fixture()
 def args() -> Configuration:
     args = Configuration()
@@ -134,7 +138,7 @@ def test_download_submission_hash_exists(
     tmp_path: Path,
     capsys: pytest.CaptureFixture,
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""
@@ -155,7 +159,7 @@ def test_download_submission_hash_exists(
 def test_download_submission_file_exists(
     downloader_mock: MagicMock, reddit_instance: praw.Reddit, tmp_path: Path, capsys: pytest.CaptureFixture
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""
@@ -202,7 +206,7 @@ def test_download_submission_min_score_above(
     tmp_path: Path,
     capsys: pytest.CaptureFixture,
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""
@@ -226,7 +230,7 @@ def test_download_submission_min_score_below(
     tmp_path: Path,
     capsys: pytest.CaptureFixture,
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""
@@ -250,7 +254,7 @@ def test_download_submission_max_score_below(
     tmp_path: Path,
     capsys: pytest.CaptureFixture,
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""
@@ -274,7 +278,7 @@ def test_download_submission_max_score_above(
     tmp_path: Path,
     capsys: pytest.CaptureFixture,
 ):
-    setup_logging(3)
+    add_console_handler()
     downloader_mock.reddit_instance = reddit_instance
     downloader_mock.download_filter.check_url.return_value = True
     downloader_mock.args.folder_scheme = ""

From 0f94c98733f191255f5b85f5cb17162b184495cc Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 9 Jan 2023 12:48:24 -0500
Subject: [PATCH 59/66] Account for new gallery url

Coverage for gallery urls
---
 bdfr/site_downloaders/imgur.py       |  9 ++--
 tests/site_downloaders/test_imgur.py | 63 +++++++---------------------
 2 files changed, 21 insertions(+), 51 deletions(-)

diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index 537ab64..4af62b1 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -42,9 +42,12 @@ class Imgur(BaseDownloader):
     @staticmethod
     def _get_data(link: str) -> dict:
         try:
-            imgur_id = re.match(r".*/(.*?)(_d)?(\..{0,})?$", link).group(1)
-            gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else ""
-            link = f"https://imgur.com/{gallery}{imgur_id}"
+            if re.search(r".*/(.*?)(gallery/|a/)", link):
+                imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1)
+                link = f"https://imgur.com/a/{imgur_id}"
+            else:
+                imgur_id = re.match(r".*/(.*?)(_d)?(\..{0,})?$", link).group(1)
+                link = f"https://imgur.com/{imgur_id}"
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Imgur ID from {link}")
 
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 7f587c9..9f8e6a7 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -109,10 +109,7 @@ def test_imgur_extension_validation_bad(test_extension: str):
     ("test_url", "expected_hashes"),
     (
         ("https://imgur.com/a/xWZsDDP", ("f551d6e6b0fef2ce909767338612e31b",)),
-        (
-            "https://imgur.com/gallery/IjJJdlC",
-            ("740b006cf9ec9d6f734b6e8f5130bdab",),
-        ),
+        ("https://imgur.com/gallery/IjJJdlC", ("740b006cf9ec9d6f734b6e8f5130bdab",)),
         (
             "https://imgur.com/a/dcc84Gt",
             (
@@ -130,50 +127,20 @@ def test_imgur_extension_validation_bad(test_extension: str):
                 "fb6c913d721c0bbb96aa65d7f560d385",
             ),
         ),
-        (
-            "https://i.imgur.com/lFJai6i.gifv",
-            ("01a6e79a30bec0e644e5da12365d5071",),
-        ),
-        (
-            "https://i.imgur.com/ywSyILa.gifv?",
-            ("56d4afc32d2966017c38d98568709b45",),
-        ),
-        (
-            "https://imgur.com/ubYwpbk.GIFV",
-            ("d4a774aac1667783f9ed3a1bd02fac0c",),
-        ),
-        (
-            "https://i.imgur.com/j1CNCZY.gifv",
-            ("58e7e6d972058c18b7ecde910ca147e3",),
-        ),
-        (
-            "https://i.imgur.com/uTvtQsw.gifv",
-            ("46c86533aa60fc0e09f2a758513e3ac2",),
-        ),
-        (
-            "https://i.imgur.com/OGeVuAe.giff",
-            ("77389679084d381336f168538793f218",),
-        ),
-        (
-            "https://i.imgur.com/OGeVuAe.gift",
-            ("77389679084d381336f168538793f218",),
-        ),
-        (
-            "https://i.imgur.com/3SKrQfK.jpg?1",
-            ("aa299e181b268578979cad176d1bd1d0",),
-        ),
-        (
-            "https://i.imgur.com/cbivYRW.jpg?3",
-            ("7ec6ceef5380cb163a1d498c359c51fd",),
-        ),
-        (
-            "http://i.imgur.com/s9uXxlq.jpg?5.jpg",
-            ("338de3c23ee21af056b3a7c154e2478f",),
-        ),
-        (
-            "https://i.imgur.com/2TtN68l_d.webp",
-            ("6569ab9ad9fa68d93f6b408f112dd741",),
-        ),
+        ("https://i.imgur.com/lFJai6i.gifv", ("01a6e79a30bec0e644e5da12365d5071",)),
+        ("https://i.imgur.com/ywSyILa.gifv?", ("56d4afc32d2966017c38d98568709b45",)),
+        ("https://imgur.com/ubYwpbk.GIFV", ("d4a774aac1667783f9ed3a1bd02fac0c",)),
+        ("https://i.imgur.com/j1CNCZY.gifv", ("58e7e6d972058c18b7ecde910ca147e3",)),
+        ("https://i.imgur.com/uTvtQsw.gifv", ("46c86533aa60fc0e09f2a758513e3ac2",)),
+        ("https://i.imgur.com/OGeVuAe.giff", ("77389679084d381336f168538793f218",)),
+        ("https://i.imgur.com/OGeVuAe.gift", ("77389679084d381336f168538793f218",)),
+        ("https://i.imgur.com/3SKrQfK.jpg?1", ("aa299e181b268578979cad176d1bd1d0",)),
+        ("https://i.imgur.com/cbivYRW.jpg?3", ("7ec6ceef5380cb163a1d498c359c51fd",)),
+        ("http://i.imgur.com/s9uXxlq.jpg?5.jpg", ("338de3c23ee21af056b3a7c154e2478f",)),
+        ("https://i.imgur.com/2TtN68l_d.webp", ("6569ab9ad9fa68d93f6b408f112dd741",)),
+        ("https://imgur.com/a/1qzfWtY/gifv", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
+        ("https://imgur.com/a/1qzfWtY/mp4", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
+        ("https://imgur.com/a/1qzfWtY/spqr", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
     ),
 )
 def test_find_resources(test_url: str, expected_hashes: list[str]):

From 8e60a12517929d27b34691f3874ef4272f87122c Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Mon, 9 Jan 2023 15:34:14 -0500
Subject: [PATCH 60/66] Imgur thumbnail coverage

Coverage for links posted to thumbnail variations.
---
 bdfr/site_downloaders/imgur.py       | 11 ++++++++---
 tests/site_downloaders/test_imgur.py |  1 +
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index 4af62b1..294209d 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -44,10 +44,15 @@ class Imgur(BaseDownloader):
         try:
             if re.search(r".*/(.*?)(gallery/|a/)", link):
                 imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1)
-                link = f"https://imgur.com/a/{imgur_id}"
             else:
-                imgur_id = re.match(r".*/(.*?)(_d)?(\..{0,})?$", link).group(1)
-                link = f"https://imgur.com/{imgur_id}"
+                imgur_id = re.match(r".*/(.*?)(?:_d)?(?:\..{0,})?$", link).group(1)
+            gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else ""
+            if len(imgur_id) > 7:
+                if imgur_id.endswith(("s", "b", "t", "m", "l", "h")):
+                    imgur_id = imgur_id[:7]
+                else:
+                    raise SiteDownloaderError(f"Imgur ID error in link {link}")
+            link = f"https://imgur.com/{gallery}{imgur_id}"
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Imgur ID from {link}")
 
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 9f8e6a7..2e74a25 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -137,6 +137,7 @@ def test_imgur_extension_validation_bad(test_extension: str):
         ("https://i.imgur.com/3SKrQfK.jpg?1", ("aa299e181b268578979cad176d1bd1d0",)),
         ("https://i.imgur.com/cbivYRW.jpg?3", ("7ec6ceef5380cb163a1d498c359c51fd",)),
         ("http://i.imgur.com/s9uXxlq.jpg?5.jpg", ("338de3c23ee21af056b3a7c154e2478f",)),
+        ("http://i.imgur.com/s9uXxlqb.jpg", ("338de3c23ee21af056b3a7c154e2478f",)),
         ("https://i.imgur.com/2TtN68l_d.webp", ("6569ab9ad9fa68d93f6b408f112dd741",)),
         ("https://imgur.com/a/1qzfWtY/gifv", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
         ("https://imgur.com/a/1qzfWtY/mp4", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),

From 5fbe64dc7128237329d6c56bc96ded954d9ccd28 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 21 Jan 2023 17:36:56 -0500
Subject: [PATCH 61/66] Move Imgur to API

Moves Imgur to use API with public Client-ID.
---
 bdfr/site_downloaders/imgur.py       | 76 +++++++--------------
 tests/site_downloaders/test_imgur.py | 98 +---------------------------
 2 files changed, 24 insertions(+), 150 deletions(-)

diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index 294209d..bfcecc0 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -5,7 +5,6 @@ import json
 import re
 from typing import Optional
 
-import bs4
 from praw.models import Submission
 
 from bdfr.exceptions import SiteDownloaderError
@@ -23,73 +22,42 @@ class Imgur(BaseDownloader):
         self.raw_data = self._get_data(self.post.url)
 
         out = []
-        if "album_images" in self.raw_data:
-            images = self.raw_data["album_images"]
-            for image in images["images"]:
-                out.append(self._compute_image_url(image))
+        if "is_album" in self.raw_data:
+            for image in self.raw_data["images"]:
+                if "mp4" in image:
+                    out.append(Resource(self.post, image["mp4"], Resource.retry_download(image["mp4"])))
+                else:
+                    out.append(Resource(self.post, image["link"], Resource.retry_download(image["link"])))
         else:
-            out.append(self._compute_image_url(self.raw_data))
+            if "mp4" in self.raw_data:
+                out.append(Resource(self.post, self.raw_data["mp4"], Resource.retry_download(self.raw_data["mp4"])))
+            else:
+                out.append(Resource(self.post, self.raw_data["link"], Resource.retry_download(self.raw_data["link"])))
         return out
 
-    def _compute_image_url(self, image: dict) -> Resource:
-        ext = self._validate_extension(image["ext"])
-        if image.get("prefer_video", False):
-            ext = ".mp4"
-
-        image_url = "https://i.imgur.com/" + image["hash"] + ext
-        return Resource(self.post, image_url, Resource.retry_download(image_url))
-
     @staticmethod
     def _get_data(link: str) -> dict:
         try:
             if re.search(r".*/(.*?)(gallery/|a/)", link):
                 imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1)
+                link = f"https://api.imgur.com/3/album/{imgur_id}"
             else:
                 imgur_id = re.match(r".*/(.*?)(?:_d)?(?:\..{0,})?$", link).group(1)
-            gallery = "a/" if re.search(r".*/(.*?)(gallery/|a/)", link) else ""
-            if len(imgur_id) > 7:
-                if imgur_id.endswith(("s", "b", "t", "m", "l", "h")):
-                    imgur_id = imgur_id[:7]
-                else:
-                    raise SiteDownloaderError(f"Imgur ID error in link {link}")
-            link = f"https://imgur.com/{gallery}{imgur_id}"
+                link = f"https://api.imgur.com/3/image/{imgur_id}"
         except AttributeError:
             raise SiteDownloaderError(f"Could not extract Imgur ID from {link}")
 
-        res = Imgur.retrieve_url(link, cookies={"over18": "1", "postpagebeta": "0"})
-
-        soup = bs4.BeautifulSoup(res.text, "html.parser")
-        scripts = soup.find_all("script", attrs={"type": "text/javascript"})
-        scripts = [script.string.replace("\n", "") for script in scripts if script.string]
-
-        script_regex = re.compile(r"\s*\(function\(widgetFactory\)\s*{\s*widgetFactory\.mergeConfig\(\'gallery\'")
-        chosen_script = list(filter(lambda s: re.search(script_regex, s), scripts))
-        if len(chosen_script) != 1:
-            raise SiteDownloaderError(f"Could not read page source from {link}")
-
-        chosen_script = chosen_script[0]
-
-        outer_regex = re.compile(r"widgetFactory\.mergeConfig\(\'gallery\', ({.*})\);")
-        inner_regex = re.compile(r"image\s*:(.*),\s*group")
-        try:
-            image_dict = re.search(outer_regex, chosen_script).group(1)
-            image_dict = re.search(inner_regex, image_dict).group(1)
-        except AttributeError:
-            raise SiteDownloaderError("Could not find image dictionary in page source")
+        headers = {
+            "referer": "https://imgur.com/",
+            "origin": "https://imgur.com",
+            "content-type": "application/json",
+            "Authorization": "Client-ID 546c25a59c58ad7",
+        }
+        res = Imgur.retrieve_url(link, headers=headers)
 
         try:
-            image_dict = json.loads(image_dict)
+            image_dict = json.loads(res.text)
         except json.JSONDecodeError as e:
-            raise SiteDownloaderError(f"Could not parse received dict as JSON: {e}")
+            raise SiteDownloaderError(f"Could not parse received response as JSON: {e}")
 
-        return image_dict
-
-    @staticmethod
-    def _validate_extension(extension_suffix: str) -> str:
-        extension_suffix = re.sub(r"\?.*", "", extension_suffix)
-        possible_extensions = (".jpg", ".png", ".mp4", ".gif")
-        selection = [ext for ext in possible_extensions if ext == extension_suffix]
-        if len(selection) == 1:
-            return selection[0]
-        else:
-            raise SiteDownloaderError(f'"{extension_suffix}" is not recognized as a valid extension for Imgur')
+        return image_dict["data"]
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 2e74a25..744488b 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -5,105 +5,10 @@ from unittest.mock import Mock
 
 import pytest
 
-from bdfr.exceptions import SiteDownloaderError
 from bdfr.resource import Resource
 from bdfr.site_downloaders.imgur import Imgur
 
 
-@pytest.mark.online
-@pytest.mark.parametrize(
-    ("test_url", "expected_gen_dict", "expected_image_dict"),
-    (
-        (
-            "https://imgur.com/a/xWZsDDP",
-            {"num_images": "1", "id": "xWZsDDP", "hash": "xWZsDDP"},
-            [{"hash": "ypa8YfS", "title": "", "ext": ".png", "animated": False}],
-        ),
-        (
-            "https://imgur.com/gallery/IjJJdlC",
-            {"num_images": 1, "id": 384898055, "hash": "IjJJdlC"},
-            [
-                {
-                    "hash": "CbbScDt",
-                    "description": "watch when he gets it",
-                    "ext": ".gif",
-                    "animated": True,
-                    "has_sound": False,
-                }
-            ],
-        ),
-        (
-            "https://imgur.com/a/dcc84Gt",
-            {"num_images": "4", "id": "dcc84Gt", "hash": "dcc84Gt"},
-            [
-                {"hash": "ylx0Kle", "ext": ".jpg", "title": ""},
-                {"hash": "TdYfKbK", "ext": ".jpg", "title": ""},
-                {"hash": "pCxGbe8", "ext": ".jpg", "title": ""},
-                {"hash": "TSAkikk", "ext": ".jpg", "title": ""},
-            ],
-        ),
-        (
-            "https://m.imgur.com/a/py3RW0j",
-            {
-                "num_images": "1",
-                "id": "py3RW0j",
-                "hash": "py3RW0j",
-            },
-            [{"hash": "K24eQmK", "has_sound": False, "ext": ".jpg"}],
-        ),
-    ),
-)
-def test_get_data_album(test_url: str, expected_gen_dict: dict, expected_image_dict: list[dict]):
-    result = Imgur._get_data(test_url)
-    assert all([result.get(key) == expected_gen_dict[key] for key in expected_gen_dict.keys()])
-
-    # Check if all the keys from the test dict are correct in at least one of the album entries
-    assert any(
-        [
-            all([image.get(key) == image_dict[key] for key in image_dict.keys()])
-            for image_dict in expected_image_dict
-            for image in result["album_images"]["images"]
-        ]
-    )
-
-
-@pytest.mark.online
-@pytest.mark.parametrize(
-    ("test_url", "expected_image_dict"),
-    (
-        ("https://i.imgur.com/dLk3FGY.gifv", {"hash": "dLk3FGY", "title": "", "ext": ".mp4", "animated": True}),
-        (
-            "https://imgur.com/65FqTpT.gifv",
-            {"hash": "65FqTpT", "title": "", "description": "", "animated": True, "mimetype": "video/mp4"},
-        ),
-    ),
-)
-def test_get_data_gif(test_url: str, expected_image_dict: dict):
-    result = Imgur._get_data(test_url)
-    assert all([result.get(key) == expected_image_dict[key] for key in expected_image_dict.keys()])
-
-
-@pytest.mark.parametrize("test_extension", (".gif", ".png", ".jpg", ".mp4"))
-def test_imgur_extension_validation_good(test_extension: str):
-    result = Imgur._validate_extension(test_extension)
-    assert result == test_extension
-
-
-@pytest.mark.parametrize(
-    "test_extension",
-    (
-        ".jpeg",
-        "bad",
-        ".avi",
-        ".test",
-        ".flac",
-    ),
-)
-def test_imgur_extension_validation_bad(test_extension: str):
-    with pytest.raises(SiteDownloaderError):
-        Imgur._validate_extension(test_extension)
-
-
 @pytest.mark.online
 @pytest.mark.parametrize(
     ("test_url", "expected_hashes"),
@@ -130,7 +35,7 @@ def test_imgur_extension_validation_bad(test_extension: str):
         ("https://i.imgur.com/lFJai6i.gifv", ("01a6e79a30bec0e644e5da12365d5071",)),
         ("https://i.imgur.com/ywSyILa.gifv?", ("56d4afc32d2966017c38d98568709b45",)),
         ("https://imgur.com/ubYwpbk.GIFV", ("d4a774aac1667783f9ed3a1bd02fac0c",)),
-        ("https://i.imgur.com/j1CNCZY.gifv", ("58e7e6d972058c18b7ecde910ca147e3",)),
+        ("https://i.imgur.com/j1CNCZY.gifv", ("ed63d7062bc32edaeea8b53f876a307c",)),
         ("https://i.imgur.com/uTvtQsw.gifv", ("46c86533aa60fc0e09f2a758513e3ac2",)),
         ("https://i.imgur.com/OGeVuAe.giff", ("77389679084d381336f168538793f218",)),
         ("https://i.imgur.com/OGeVuAe.gift", ("77389679084d381336f168538793f218",)),
@@ -142,6 +47,7 @@ def test_imgur_extension_validation_bad(test_extension: str):
         ("https://imgur.com/a/1qzfWtY/gifv", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
         ("https://imgur.com/a/1qzfWtY/mp4", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
         ("https://imgur.com/a/1qzfWtY/spqr", ("65fbc7ba5c3ed0e3af47c4feef4d3735",)),
+        ("https://i.imgur.com/expO7Rc.gifv", ("e309f98158fc98072eb2ae68f947f421",)),
     ),
 )
 def test_find_resources(test_url: str, expected_hashes: list[str]):

From 804d0eb661f8229b09c988bbbbca23b269dd04d9 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 21 Jan 2023 22:01:18 -0500
Subject: [PATCH 62/66] Fix failing test

Replace deleted user in test.
---
 tests/test_connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_connector.py b/tests/test_connector.py
index d300b45..fdd3dbe 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -336,7 +336,7 @@ def test_get_multireddits_public(
     (
         ("danigirl3694", 10),
         ("danigirl3694", 50),
-        ("CapitanHam", None),
+        ("nasa", None),
     ),
 )
 def test_get_user_submissions(test_user: str, limit: int, downloader_mock: MagicMock, reddit_instance: praw.Reddit):

From 105ceaf386cf782202eea4db1b1e3487f0a127a7 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Wed, 25 Jan 2023 04:43:06 -0500
Subject: [PATCH 63/66] More Redgifs coverage

---
 bdfr/site_downloaders/redgifs.py       | 2 +-
 tests/site_downloaders/test_redgifs.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bdfr/site_downloaders/redgifs.py b/bdfr/site_downloaders/redgifs.py
index 205674a..9c469bc 100644
--- a/bdfr/site_downloaders/redgifs.py
+++ b/bdfr/site_downloaders/redgifs.py
@@ -27,7 +27,7 @@ class Redgifs(BaseDownloader):
         try:
             if url.endswith("/"):
                 url = url.removesuffix("/")
-            redgif_id = re.match(r".*/(.*?)(?:\?.*|\..{0,})?$", url).group(1).lower()
+            redgif_id = re.match(r".*/(.*?)(?:#.*|\?.*|\..{0,})?$", url).group(1).lower()
             if redgif_id.endswith("-mobile"):
                 redgif_id = redgif_id.removesuffix("-mobile")
         except AttributeError:
diff --git a/tests/site_downloaders/test_redgifs.py b/tests/site_downloaders/test_redgifs.py
index 5f4f6fc..9d1a7f5 100644
--- a/tests/site_downloaders/test_redgifs.py
+++ b/tests/site_downloaders/test_redgifs.py
@@ -18,6 +18,7 @@ from bdfr.site_downloaders.redgifs import Redgifs
         ("https://www.redgifs.com/watch/marriedcrushingcob?rel=u%3Akokiri.girl%3Bo%3Arecent", "marriedcrushingcob"),
         ("https://thumbs4.redgifs.com/DismalIgnorantDrongo.mp4", "dismalignorantdrongo"),
         ("https://thumbs4.redgifs.com/DismalIgnorantDrongo-mobile.mp4", "dismalignorantdrongo"),
+        ("https://v3.redgifs.com/watch/newilliteratemeerkat#rel=user%3Atastynova", "newilliteratemeerkat"),
     ),
 )
 def test_get_id(test_url: str, expected: str):

From cf5f7bfd16b924f2c7fef45c51553f7bb5e24119 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Wed, 25 Jan 2023 22:23:59 -0500
Subject: [PATCH 64/66] pep585 and pathlib updates

---
 bdfr/archiver.py                          |  8 +++++---
 bdfr/cloner.py                            |  2 +-
 bdfr/completion.py                        |  6 +++---
 bdfr/configuration.py                     |  2 +-
 bdfr/connector.py                         |  6 +++---
 bdfr/downloader.py                        |  2 +-
 bdfr/oauth2.py                            |  2 +-
 bdfr/resource.py                          |  3 ++-
 bdfr/site_downloaders/download_factory.py |  3 +--
 bdfr/site_downloaders/erome.py            |  3 ++-
 bdfr/site_downloaders/youtube.py          |  3 ++-
 tests/test_connector.py                   |  2 +-
 tests/test_downloader.py                  |  5 ++---
 tests/test_file_name_formatter.py         | 12 ++++++------
 14 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/bdfr/archiver.py b/bdfr/archiver.py
index 88136a8..52b4649 100644
--- a/bdfr/archiver.py
+++ b/bdfr/archiver.py
@@ -4,8 +4,10 @@
 import json
 import logging
 import re
+from collections.abc import Iterable, Iterator
+from pathlib import Path
 from time import sleep
-from typing import Iterable, Iterator, Union
+from typing import Union
 
 import dict2xml
 import praw.models
@@ -108,13 +110,13 @@ class Archiver(RedditConnector):
 
     def _write_entry_yaml(self, entry: BaseArchiveEntry):
         resource = Resource(entry.source, "", lambda: None, ".yaml")
-        content = yaml.dump(entry.compile())
+        content = yaml.safe_dump(entry.compile())
         self._write_content_to_disk(resource, content)
 
     def _write_content_to_disk(self, resource: Resource, content: str):
         file_path = self.file_name_formatter.format_path(resource, self.download_directory)
         file_path.parent.mkdir(exist_ok=True, parents=True)
-        with open(file_path, "w", encoding="utf-8") as file:
+        with Path(file_path).open(mode="w", encoding="utf-8") as file:
             logger.debug(
                 f"Writing entry {resource.source_submission.id} to file in {resource.extension[1:].upper()}"
                 f" format at {file_path}"
diff --git a/bdfr/cloner.py b/bdfr/cloner.py
index c31f6cc..df71c28 100644
--- a/bdfr/cloner.py
+++ b/bdfr/cloner.py
@@ -2,8 +2,8 @@
 # -*- coding: utf-8 -*-
 
 import logging
+from collections.abc import Iterable
 from time import sleep
-from typing import Iterable
 
 import prawcore
 
diff --git a/bdfr/completion.py b/bdfr/completion.py
index 7b38322..8ec4e2c 100644
--- a/bdfr/completion.py
+++ b/bdfr/completion.py
@@ -23,7 +23,7 @@ class Completion:
                 Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "bash_source"
-                with open(comp_dir + point, "w") as file:
+                with Path(comp_dir + point).open(mode="w") as file:
                     file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
                     print(f"Bash completion for {point} written to {comp_dir}{point}")
         if self.shell in ("all", "fish"):
@@ -33,7 +33,7 @@ class Completion:
                 Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "fish_source"
-                with open(comp_dir + point + ".fish", "w") as file:
+                with Path(comp_dir + point + ".fish").open(mode="w") as file:
                     file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
                     print(f"Fish completion for {point} written to {comp_dir}{point}.fish")
         if self.shell in ("all", "zsh"):
@@ -43,7 +43,7 @@ class Completion:
                 Path(comp_dir).mkdir(parents=True, exist_ok=True)
             for point in self.entry_points:
                 self.env[f"_{point.upper().replace('-', '_')}_COMPLETE"] = "zsh_source"
-                with open(comp_dir + "_" + point, "w") as file:
+                with Path(comp_dir + "_" + point).open(mode="w") as file:
                     file.write(subprocess.run([point], env=self.env, capture_output=True, text=True).stdout)
                     print(f"Zsh completion for {point} written to {comp_dir}_{point}")
 
diff --git a/bdfr/configuration.py b/bdfr/configuration.py
index 78ae12e..05fc27e 100644
--- a/bdfr/configuration.py
+++ b/bdfr/configuration.py
@@ -79,7 +79,7 @@ class Configuration(Namespace):
             return
         with yaml_file_loc.open() as file:
             try:
-                opts = yaml.load(file, Loader=yaml.FullLoader)
+                opts = yaml.safe_load(file)
             except yaml.YAMLError as e:
                 logger.error(f"Could not parse YAML options file: {e}")
                 return
diff --git a/bdfr/connector.py b/bdfr/connector.py
index 89339f0..77a4a71 100644
--- a/bdfr/connector.py
+++ b/bdfr/connector.py
@@ -10,11 +10,11 @@ import re
 import shutil
 import socket
 from abc import ABCMeta, abstractmethod
+from collections.abc import Callable, Iterable, Iterator
 from datetime import datetime
 from enum import Enum, auto
 from pathlib import Path
 from time import sleep
-from typing import Callable, Iterable, Iterator
 
 import appdirs
 import praw
@@ -119,7 +119,7 @@ class RedditConnector(metaclass=ABCMeta):
             )
             logger.debug(f"Setting filename restriction scheme to '{self.args.filename_restriction_scheme}'")
         # Update config on disk
-        with open(self.config_location, "w") as file:
+        with Path(self.config_location).open(mode="w") as file:
             self.cfg_parser.write(file)
 
     def parse_disabled_modules(self):
@@ -143,7 +143,7 @@ class RedditConnector(metaclass=ABCMeta):
                 )
                 token = oauth2_authenticator.retrieve_new_token()
                 self.cfg_parser["DEFAULT"]["user_token"] = token
-                with open(self.config_location, "w") as file:
+                with Path(self.config_location).open(mode="w") as file:
                     self.cfg_parser.write(file, True)
             token_manager = OAuth2TokenManager(self.cfg_parser, self.config_location)
 
diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index 31c839d..0a5177e 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -5,11 +5,11 @@ import hashlib
 import logging.handlers
 import os
 import time
+from collections.abc import Iterable
 from datetime import datetime
 from multiprocessing import Pool
 from pathlib import Path
 from time import sleep
-from typing import Iterable
 
 import praw
 import praw.exceptions
diff --git a/bdfr/oauth2.py b/bdfr/oauth2.py
index 28b956a..ead0553 100644
--- a/bdfr/oauth2.py
+++ b/bdfr/oauth2.py
@@ -103,6 +103,6 @@ class OAuth2TokenManager(praw.reddit.BaseTokenManager):
 
     def post_refresh_callback(self, authorizer: praw.reddit.Authorizer):
         self.config.set("DEFAULT", "user_token", authorizer.refresh_token)
-        with open(self.config_location, "w") as file:
+        with Path(self.config_location).open(mode="w") as file:
             self.config.write(file, True)
         logger.log(9, f"Written OAuth2 token from authoriser to {self.config_location}")
diff --git a/bdfr/resource.py b/bdfr/resource.py
index bd3ae88..37fc521 100644
--- a/bdfr/resource.py
+++ b/bdfr/resource.py
@@ -6,7 +6,8 @@ import logging
 import re
 import time
 import urllib.parse
-from typing import Callable, Optional
+from collections.abc import Callable
+from typing import Optional
 
 import _hashlib
 import requests
diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index 719d564..9006681 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -3,7 +3,6 @@
 
 import re
 import urllib.parse
-from typing import Type
 
 from bdfr.exceptions import NotADownloadableLinkError
 from bdfr.site_downloaders.base_downloader import BaseDownloader
@@ -24,7 +23,7 @@ from bdfr.site_downloaders.youtube import Youtube
 
 class DownloadFactory:
     @staticmethod
-    def pull_lever(url: str) -> Type[BaseDownloader]:
+    def pull_lever(url: str) -> type[BaseDownloader]:
         sanitised_url = DownloadFactory.sanitise_url(url)
         if re.match(r"(i\.|m\.)?imgur", sanitised_url):
             return Imgur
diff --git a/bdfr/site_downloaders/erome.py b/bdfr/site_downloaders/erome.py
index bf139d2..894e470 100644
--- a/bdfr/site_downloaders/erome.py
+++ b/bdfr/site_downloaders/erome.py
@@ -3,7 +3,8 @@
 
 import logging
 import re
-from typing import Callable, Optional
+from collections.abc import Callable
+from typing import Optional
 
 import bs4
 from praw.models import Submission
diff --git a/bdfr/site_downloaders/youtube.py b/bdfr/site_downloaders/youtube.py
index f0c0677..306d2e1 100644
--- a/bdfr/site_downloaders/youtube.py
+++ b/bdfr/site_downloaders/youtube.py
@@ -3,8 +3,9 @@
 
 import logging
 import tempfile
+from collections.abc import Callable
 from pathlib import Path
-from typing import Callable, Optional
+from typing import Optional
 
 import yt_dlp
 from praw.models import Submission
diff --git a/tests/test_connector.py b/tests/test_connector.py
index fdd3dbe..9eabac7 100644
--- a/tests/test_connector.py
+++ b/tests/test_connector.py
@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
+from collections.abc import Iterator
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Iterator
 from unittest.mock import MagicMock
 
 import praw
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index b78a81c..ebf8218 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 import logging
-import os
 import re
 from pathlib import Path
 from unittest.mock import MagicMock, patch
@@ -118,12 +117,12 @@ def test_file_creation_date(
     RedditDownloader._download_submission(downloader_mock, submission)
 
     for file_path in Path(tmp_path).iterdir():
-        file_stats = os.stat(file_path)
+        file_stats = Path(file_path).stat()
         assert file_stats.st_mtime == test_creation_date
 
 
 def test_search_existing_files():
-    results = RedditDownloader.scan_existing_files(Path("."))
+    results = RedditDownloader.scan_existing_files(Path())
     assert len(results.keys()) != 0
 
 
diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py
index fb34a53..b23b0b1 100644
--- a/tests/test_file_name_formatter.py
+++ b/tests/test_file_name_formatter.py
@@ -6,7 +6,7 @@ import sys
 import unittest.mock
 from datetime import datetime
 from pathlib import Path
-from typing import Optional, Type, Union
+from typing import Optional, Union
 from unittest.mock import MagicMock
 
 import praw.models
@@ -222,7 +222,7 @@ def test_format_multiple_resources():
         new_mock.source_submission.__class__ = praw.models.Submission
         mocks.append(new_mock)
     test_formatter = FileNameFormatter("{TITLE}", "", "ISO")
-    results = test_formatter.format_resource_paths(mocks, Path("."))
+    results = test_formatter.format_resource_paths(mocks, Path())
     results = set([str(res[0].name) for res in results])
     expected = {"test_1.png", "test_2.png", "test_3.png", "test_4.png"}
     assert results == expected
@@ -238,7 +238,7 @@ def test_format_multiple_resources():
     ),
 )
 def test_limit_filename_length(test_filename: str, test_ending: str, test_formatter: FileNameFormatter):
-    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
+    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path())
     assert len(result.name) <= 255
     assert len(result.name.encode("utf-8")) <= 255
     assert len(str(result)) <= FileNameFormatter.find_max_path_length()
@@ -262,7 +262,7 @@ def test_limit_filename_length(test_filename: str, test_ending: str, test_format
 def test_preserve_id_append_when_shortening(
     test_filename: str, test_ending: str, expected_end: str, test_formatter: FileNameFormatter
 ):
-    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
+    result = test_formatter.limit_file_name_length(test_filename, test_ending, Path())
     assert len(result.name) <= 255
     assert len(result.name.encode("utf-8")) <= 255
     assert result.name.endswith(expected_end)
@@ -509,13 +509,13 @@ def test_windows_max_path(tmp_path: Path):
 )
 def test_name_submission(
     test_reddit_id: str,
-    test_downloader: Type[BaseDownloader],
+    test_downloader: type[BaseDownloader],
     expected_names: set[str],
     reddit_instance: praw.reddit.Reddit,
 ):
     test_submission = reddit_instance.submission(id=test_reddit_id)
     test_resources = test_downloader(test_submission).find_resources()
     test_formatter = FileNameFormatter("{TITLE}", "", "")
-    results = test_formatter.format_resource_paths(test_resources, Path("."))
+    results = test_formatter.format_resource_paths(test_resources, Path())
     results = set([r[0].name for r in results])
     assert results == expected_names

From 63b0607f58cc0bc404b044d9a55b1bb4c71d8789 Mon Sep 17 00:00:00 2001
From: OMEGARAZER <869111+OMEGARAZER@users.noreply.github.com>
Date: Sat, 28 Jan 2023 02:04:06 -0500
Subject: [PATCH 65/66] bugbear B007

---
 bdfr/downloader.py                | 2 +-
 tests/test_file_name_formatter.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bdfr/downloader.py b/bdfr/downloader.py
index 0a5177e..20984e6 100644
--- a/bdfr/downloader.py
+++ b/bdfr/downloader.py
@@ -156,7 +156,7 @@ class RedditDownloader(RedditConnector):
     @staticmethod
     def scan_existing_files(directory: Path) -> dict[str, Path]:
         files = []
-        for (dirpath, dirnames, filenames) in os.walk(directory):
+        for (dirpath, _dirnames, filenames) in os.walk(directory):
             files.extend([Path(dirpath, file) for file in filenames])
         logger.info(f"Calculating hashes for {len(files)} files")
 
diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py
index b23b0b1..f456415 100644
--- a/tests/test_file_name_formatter.py
+++ b/tests/test_file_name_formatter.py
@@ -214,7 +214,7 @@ def test_format_full_with_index_suffix(
 
 def test_format_multiple_resources():
     mocks = []
-    for i in range(1, 5):
+    for _i in range(1, 5):
         new_mock = MagicMock()
         new_mock.url = "https://example.com/test.png"
         new_mock.extension = ".png"

From 0e23dcb8ad64a09c4758801bd070e32fbfe07c32 Mon Sep 17 00:00:00 2001
From: Soulsuck24 <79275800+Soulsuck24@users.noreply.github.com>
Date: Mon, 30 Jan 2023 14:52:08 -0500
Subject: [PATCH 66/66] Gfycat/Redgifs coverage

Coverage for direct gfycat links that redirect to redgifs. The redirect through the sites themselves are broken but this fixes that.

Coverage for o.imgur links and incorrect capitalisation of domains in download_factory.

Changed tests for direct as gfycat is handled by the gfycat downloader.

fix pornhub test as the previous video was removed.
---
 bdfr/site_downloaders/download_factory.py       |  8 ++++----
 bdfr/site_downloaders/gfycat.py                 |  2 +-
 bdfr/site_downloaders/imgur.py                  |  2 ++
 tests/site_downloaders/test_direct.py           |  7 +++++--
 tests/site_downloaders/test_download_factory.py |  1 +
 tests/site_downloaders/test_gfycat.py           | 11 ++++++++++-
 tests/site_downloaders/test_imgur.py            |  2 ++
 tests/site_downloaders/test_pornhub.py          |  2 +-
 8 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/bdfr/site_downloaders/download_factory.py b/bdfr/site_downloaders/download_factory.py
index 9006681..578f4d3 100644
--- a/bdfr/site_downloaders/download_factory.py
+++ b/bdfr/site_downloaders/download_factory.py
@@ -24,11 +24,13 @@ from bdfr.site_downloaders.youtube import Youtube
 class DownloadFactory:
     @staticmethod
     def pull_lever(url: str) -> type[BaseDownloader]:
-        sanitised_url = DownloadFactory.sanitise_url(url)
-        if re.match(r"(i\.|m\.)?imgur", sanitised_url):
+        sanitised_url = DownloadFactory.sanitise_url(url).lower()
+        if re.match(r"(i\.|m\.|o\.)?imgur", sanitised_url):
             return Imgur
         elif re.match(r"(i\.|thumbs\d\.|v\d\.)?(redgifs|gifdeliverynetwork)", sanitised_url):
             return Redgifs
+        elif re.match(r"(thumbs\.|giant\.)?gfycat\.", sanitised_url):
+            return Gfycat
         elif re.match(r".*/.*\.[a-zA-Z34]{3,4}(\?[\w;&=]*)?$", sanitised_url) and not DownloadFactory.is_web_resource(
             sanitised_url
         ):
@@ -41,8 +43,6 @@ class DownloadFactory:
             return Gallery
         elif re.match(r"patreon\.com.*", sanitised_url):
             return Gallery
-        elif re.match(r"gfycat\.", sanitised_url):
-            return Gfycat
         elif re.match(r"reddit\.com/r/", sanitised_url):
             return SelfPost
         elif re.match(r"(m\.)?youtu\.?be", sanitised_url):
diff --git a/bdfr/site_downloaders/gfycat.py b/bdfr/site_downloaders/gfycat.py
index d7c60ca..4524689 100644
--- a/bdfr/site_downloaders/gfycat.py
+++ b/bdfr/site_downloaders/gfycat.py
@@ -23,7 +23,7 @@ class Gfycat(Redgifs):
 
     @staticmethod
     def _get_link(url: str) -> set[str]:
-        gfycat_id = re.match(r".*/(.*?)/?$", url).group(1)
+        gfycat_id = re.match(r".*/(.*?)(?:/?|-.*|\..{3-4})$", url).group(1)
         url = "https://gfycat.com/" + gfycat_id
 
         response = Gfycat.retrieve_url(url)
diff --git a/bdfr/site_downloaders/imgur.py b/bdfr/site_downloaders/imgur.py
index bfcecc0..26c76e6 100644
--- a/bdfr/site_downloaders/imgur.py
+++ b/bdfr/site_downloaders/imgur.py
@@ -38,6 +38,8 @@ class Imgur(BaseDownloader):
     @staticmethod
     def _get_data(link: str) -> dict:
         try:
+            if link.endswith("/"):
+                link = link.removesuffix("/")
             if re.search(r".*/(.*?)(gallery/|a/)", link):
                 imgur_id = re.match(r".*/(?:gallery/|a/)(.*?)(?:/.*)?$", link).group(1)
                 link = f"https://api.imgur.com/3/album/{imgur_id}"
diff --git a/tests/site_downloaders/test_direct.py b/tests/site_downloaders/test_direct.py
index 14190ee..c4279cd 100644
--- a/tests/site_downloaders/test_direct.py
+++ b/tests/site_downloaders/test_direct.py
@@ -13,8 +13,11 @@ from bdfr.site_downloaders.direct import Direct
 @pytest.mark.parametrize(
     ("test_url", "expected_hash"),
     (
-        ("https://giant.gfycat.com/DefinitiveCanineCrayfish.mp4", "48f9bd4dbec1556d7838885612b13b39"),
-        ("https://giant.gfycat.com/DazzlingSilkyIguana.mp4", "808941b48fc1e28713d36dd7ed9dc648"),
+        ("https://i.redd.it/q6ebualjxzea1.jpg", "6ec154859c777cb401132bb991cb3635"),
+        (
+            "https://file-examples.com/wp-content/uploads/2017/11/file_example_MP3_700KB.mp3",
+            "3caa342e241ddb7d76fd24a834094101",
+        ),
     ),
 )
 def test_download_resource(test_url: str, expected_hash: str):
diff --git a/tests/site_downloaders/test_download_factory.py b/tests/site_downloaders/test_download_factory.py
index f95e609..0706c69 100644
--- a/tests/site_downloaders/test_download_factory.py
+++ b/tests/site_downloaders/test_download_factory.py
@@ -31,6 +31,7 @@ from bdfr.site_downloaders.youtube import Youtube
         ),
         ("https://i.redd.it/affyv0axd5k61.png", Direct),
         ("https://i.imgur.com/bZx1SJQ.jpg", Imgur),
+        ("https://i.Imgur.com/bZx1SJQ.jpg", Imgur),
         ("https://imgur.com/BuzvZwb.gifv", Imgur),
         ("https://imgur.com/a/MkxAzeg", Imgur),
         ("https://m.imgur.com/a/py3RW0j", Imgur),
diff --git a/tests/site_downloaders/test_gfycat.py b/tests/site_downloaders/test_gfycat.py
index 0cfb36f..2821a7e 100644
--- a/tests/site_downloaders/test_gfycat.py
+++ b/tests/site_downloaders/test_gfycat.py
@@ -15,11 +15,17 @@ from bdfr.site_downloaders.gfycat import Gfycat
     (
         ("https://gfycat.com/definitivecaninecrayfish", "https://giant.gfycat.com/DefinitiveCanineCrayfish.mp4"),
         ("https://gfycat.com/dazzlingsilkyiguana", "https://giant.gfycat.com/DazzlingSilkyIguana.mp4"),
+        ("https://gfycat.com/WearyComposedHairstreak", "https://thumbs4.redgifs.com/WearyComposedHairstreak.mp4"),
+        (
+            "https://thumbs.gfycat.com/ComposedWholeBullfrog-size_restricted.gif",
+            "https://thumbs4.redgifs.com/ComposedWholeBullfrog.mp4",
+        ),
+        ("https://giant.gfycat.com/ComposedWholeBullfrog.mp4", "https://thumbs4.redgifs.com/ComposedWholeBullfrog.mp4"),
     ),
 )
 def test_get_link(test_url: str, expected_url: str):
     result = Gfycat._get_link(test_url)
-    assert result.pop() == expected_url
+    assert expected_url in result.pop()
 
 
 @pytest.mark.online
@@ -28,6 +34,9 @@ def test_get_link(test_url: str, expected_url: str):
     (
         ("https://gfycat.com/definitivecaninecrayfish", "48f9bd4dbec1556d7838885612b13b39"),
         ("https://gfycat.com/dazzlingsilkyiguana", "808941b48fc1e28713d36dd7ed9dc648"),
+        ("https://gfycat.com/WearyComposedHairstreak", "5f82ba1ba23cc927c9fbb0c0421953a5"),
+        ("https://thumbs.gfycat.com/ComposedWholeBullfrog-size_restricted.gif", "5292343665a13b5369d889d911ae284d"),
+        ("https://giant.gfycat.com/ComposedWholeBullfrog.mp4", "5292343665a13b5369d889d911ae284d"),
     ),
 )
 def test_download_resource(test_url: str, expected_hash: str):
diff --git a/tests/site_downloaders/test_imgur.py b/tests/site_downloaders/test_imgur.py
index 744488b..46dd979 100644
--- a/tests/site_downloaders/test_imgur.py
+++ b/tests/site_downloaders/test_imgur.py
@@ -15,6 +15,7 @@ from bdfr.site_downloaders.imgur import Imgur
     (
         ("https://imgur.com/a/xWZsDDP", ("f551d6e6b0fef2ce909767338612e31b",)),
         ("https://imgur.com/gallery/IjJJdlC", ("740b006cf9ec9d6f734b6e8f5130bdab",)),
+        ("https://imgur.com/gallery/IjJJdlC/", ("740b006cf9ec9d6f734b6e8f5130bdab",)),
         (
             "https://imgur.com/a/dcc84Gt",
             (
@@ -32,6 +33,7 @@ from bdfr.site_downloaders.imgur import Imgur
                 "fb6c913d721c0bbb96aa65d7f560d385",
             ),
         ),
+        ("https://o.imgur.com/jZw9gq2.jpg", ("6d6ea9aa1d98827a05425338afe675bc",)),
         ("https://i.imgur.com/lFJai6i.gifv", ("01a6e79a30bec0e644e5da12365d5071",)),
         ("https://i.imgur.com/ywSyILa.gifv?", ("56d4afc32d2966017c38d98568709b45",)),
         ("https://imgur.com/ubYwpbk.GIFV", ("d4a774aac1667783f9ed3a1bd02fac0c",)),
diff --git a/tests/site_downloaders/test_pornhub.py b/tests/site_downloaders/test_pornhub.py
index d9971cb..894b9f8 100644
--- a/tests/site_downloaders/test_pornhub.py
+++ b/tests/site_downloaders/test_pornhub.py
@@ -14,7 +14,7 @@ from bdfr.site_downloaders.pornhub import PornHub
 @pytest.mark.slow
 @pytest.mark.parametrize(
     ("test_url", "expected_hash"),
-    (("https://www.pornhub.com/view_video.php?viewkey=ph6074c59798497", "ad52a0f4fce8f99df0abed17de1d04c7"),),
+    (("https://www.pornhub.com/view_video.php?viewkey=ph5eafee2d174ff", "d15090cbbaa8ee90500a257c7899ff84"),),
 )
 def test_hash_resources_good(test_url: str, expected_hash: str):
     test_submission = MagicMock()