CodeMirror/bulk-downloader-for-reddit
1
0
Fork 0
mirror of synced 2024-06-02 18:34:37 +12:00
Code Issues Releases Wiki Activity

Scripts testing/fixes

Browse source 
Adds Bats and Pester testing to for bash and powershell scripts

Updates powershell scripts to match bash scripts in logic

Added missing score filter lookup for powershell script
This commit is contained in:
OMEGARAZER 2023-03-01 23:13:09 -05:00
parent a16622e11e
commit 74d842e6da
No known key found for this signature in database
GPG key ID: D89925310D306E35
12 changed files with 155 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
.github/workflows/scripts-test.yml vendored Normal file
View file

@ -0,0 +1,29 @@
name: Scripts Test
on:
push:
paths:
- "scripts/*.sh"
- "scripts/*.ps1"
pull_request:
paths:
- "scripts/*.sh"
- "scripts/*.ps1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
submodules: 'true'
- name: Bats tests
run: |
cd scripts/tests/
bats/bin/bats *.bats
- name: Pester tests
shell: pwsh
run: |
cd scripts/tests/
Invoke-Pester -CI -PassThru .

26
scripts/extract_failed_ids.ps1
View file

@ -1,21 +1,13 @@
if (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
else {
Write-Host "CANNOT FIND LOG FILE"
if (($args[0] -eq $null) -or -Not (Test-Path -Path $args[0] -PathType Leaf)) {
Write-Output "CANNOT FIND LOG FILE"
Exit 1
}
if ($null -ne $args[1]) {
$output=$args[1]
Write-Host "Outputting IDs to $output"
}
else {
$output="./failed.txt"
elseif (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
Select-String -Path $file -Pattern "Could not download submission" | ForEach-Object { -split $_.Line | Select-Object -Skip 11 | Select-Object -First 1 } | ForEach-Object { $_.substring(0,$_.Length-1) } >> $output
Select-String -Path $file -Pattern "Failed to download resource" | ForEach-Object { -split $_.Line | Select-Object -Skip 14 | Select-Object -First 1 } >> $output
Select-String -Path $file -Pattern "failed to download submission" | ForEach-Object { -split $_.Line | Select-Object -Skip 13 | Select-Object -First 1 } | ForEach-Object { $_.substring(0,$_.Length-1) } >> $output
Select-String -Path $file -Pattern "Failed to write file" | ForEach-Object { -split $_.Line | Select-Object -Skip 13 | Select-Object -First 1 } >> $output
Select-String -Path $file -Pattern "skipped due to disabled module" | ForEach-Object { -split $_.Line | Select-Object -Skip 8 | Select-Object -First 1 } >> $output
Select-String -Path $file -Pattern "Could not download submission" | ForEach-Object { -split $_.Line | Select-Object -Skip 11 | Select-Object -First 1 } | ForEach-Object { $_.substring(0,$_.Length-1) }
Select-String -Path $file -Pattern "Failed to download resource" | ForEach-Object { -split $_.Line | Select-Object -Skip 14 | Select-Object -First 1 }
Select-String -Path $file -Pattern "failed to download submission" | ForEach-Object { -split $_.Line | Select-Object -Skip 13 | Select-Object -First 1 } | ForEach-Object { $_.substring(0,$_.Length-1) }
Select-String -Path $file -Pattern "Failed to write file" | ForEach-Object { -split $_.Line | Select-Object -Skip 13 | Select-Object -First 1 }
Select-String -Path $file -Pattern "skipped due to disabled module" | ForEach-Object { -split $_.Line | Select-Object -Skip 8 | Select-Object -First 1 }

2
scripts/extract_failed_ids.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash
if [ -e "$1" ]; then
if [ -e "$1" ] && [ -f "$1" ]; then
file="$1"
else
echo "CANNOT FIND LOG FILE"

27
scripts/extract_successful_ids.ps1
View file

@ -1,21 +1,14 @@
if (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
else {
Write-Host "CANNOT FIND LOG FILE"
if (($args[0] -eq $null) -or -Not (Test-Path -Path $args[0] -PathType Leaf)) {
Write-Output "CANNOT FIND LOG FILE"
Exit 1
}
if ($null -ne $args[1]) {
$output=$args[1]
Write-Host "Outputting IDs to $output"
}
else {
$output="./successful.txt"
elseif (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
Select-String -Path $file -Pattern "Downloaded submission" | ForEach-Object { -split $_.Line | Select-Object -Last 3 | Select-Object -SkipLast 2 } >> $output
Select-String -Path $file -Pattern "Resource hash" | ForEach-Object { -split $_.Line | Select-Object -Last 3 | Select-Object -SkipLast 2 } >> $output
Select-String -Path $file -Pattern "Download filter" | ForEach-Object { -split $_.Line | Select-Object -Last 4 | Select-Object -SkipLast 3 } >> $output
Select-String -Path $file -Pattern "already exists, continuing" | ForEach-Object { -split $_.Line | Select-Object -Last 4 | Select-Object -SkipLast 3 } >> $output
Select-String -Path $file -Pattern "Hard link made" | ForEach-Object { -split $_.Line | Select-Object -Last 1 } >> $output
Select-String -Path $file -Pattern "Downloaded submission" | ForEach-Object { -split $_.Line | Select-Object -Last 3 | Select-Object -SkipLast 2 }
Select-String -Path $file -Pattern "Resource hash" | ForEach-Object { -split $_.Line | Select-Object -Last 3 | Select-Object -SkipLast 2 }
Select-String -Path $file -Pattern "Download filter" | ForEach-Object { -split $_.Line | Select-Object -Last 4 | Select-Object -SkipLast 3 }
Select-String -Path $file -Pattern "already exists, continuing" | ForEach-Object { -split $_.Line | Select-Object -Last 4 | Select-Object -SkipLast 3 }
Select-String -Path $file -Pattern "Hard link made" | ForEach-Object { -split $_.Line | Select-Object -Last 1 }
Select-String -Path $file -Pattern "filtered due to score" | ForEach-Object { -split $_.Line | Select-Object -Index 8 }

2
scripts/extract_successful_ids.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash
if [ -e "$1" ]; then
if [ -e "$1" ] && [ -f "$1" ]; then
file="$1"
else
echo "CANNOT FIND LOG FILE"

14
scripts/print_summary.ps1
View file

@ -1,17 +1,9 @@
if (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
else {
if (($args[0] -eq $null) -or -Not (Test-Path -Path $args[0] -PathType Leaf)) {
Write-Host "CANNOT FIND LOG FILE"
Exit 1
}
if ($null -ne $args[1]) {
$output=$args[1]
Write-Host "Outputting IDs to $output"
}
else {
$output="./successful.txt"
elseif (Test-Path -Path $args[0] -PathType Leaf) {
$file=$args[0]
}
Write-Host -NoNewline "Downloaded submissions: "

2
scripts/print_summary.sh
View file

@ -1,6 +1,6 @@
#!/bin/bash
if [ -e "$1" ]; then
if [ -e "$1" ] && [ -f "$1" ]; then
file="$1"
else
echo "CANNOT FIND LOG FILE"

2
scripts/tests/example_logfiles/succeed_score_filter.txt
View file

@ -1,2 +1,2 @@
[2022-07-23 14:04:14,095 - bdfr.downloader - DEBUG] - Submission ljyy27 filtered due to score 15 < [50]
[2022-07-23 14:04:14,104 - bdfr.downloader - DEBUG] - Submission ljyy27 filtered due to score 16 > [1]
[2022-07-23 14:04:14,104 - bdfr.downloader - DEBUG] - Submission ljyz27 filtered due to score 16 > [1]

39
scripts/tests/extract_failed_ids.Tests.ps1 Normal file
View file

@ -0,0 +1,39 @@
Describe "extract_failed_ids" {
It "fail run no args" {
(..\extract_failed_ids.ps1) | Should -Be "CANNOT FIND LOG FILE"
}
It "fail run no logfile" {
(..\extract_failed_ids.ps1 missing.txt) | Should -Be "CANNOT FIND LOG FILE"
}
It "fail no downloader module" {
$down_error = (..\extract_failed_ids.ps1 example_logfiles\failed_no_downloader.txt)
$down_error | Should -HaveCount 3
$down_error | Should -Contain "nxv3ea"
}
It "fail resource error" {
$res_error = (..\extract_failed_ids.ps1 example_logfiles\failed_resource_error.txt)
$res_error | Should -HaveCount 1
$res_error | Should -Contain "nxv3dt"
}
It "fail site downloader error" {
$site_error = (..\extract_failed_ids.ps1 example_logfiles\failed_sitedownloader_error.txt)
$site_error | Should -HaveCount 2
$site_error | Should -Contain "nxpn0h"
}
It "fail failed file write" {
$write_error = (..\extract_failed_ids.ps1 example_logfiles\failed_write_error.txt)
$write_error | Should -HaveCount 1
$write_error | Should -Contain "nnboza"
}
It "fail disabled module" {
$disabled = (..\extract_failed_ids.ps1 example_logfiles\failed_disabled_module.txt)
$disabled | Should -HaveCount 1
$disabled | Should -Contain "m2601g"
}
}

45
scripts/tests/extract_successful_ids.Tests.ps1 Normal file
View file

@ -0,0 +1,45 @@
Describe "extract_successful_ids" {
It "fail run no args" {
(..\extract_successful_ids.ps1) | Should -Be "CANNOT FIND LOG FILE"
}
It "fail run no logfile" {
(..\extract_successful_ids.ps1 missing.txt) | Should -Be "CANNOT FIND LOG FILE"
}
It "success downloaded submission" {
$down_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_downloaded_submission.txt)
$down_success | Should -HaveCount 7
$down_success | Should -Contain "nn9cor"
}
It "success resource hash" {
$hash_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_resource_hash.txt)
$hash_success | Should -HaveCount 1
$hash_success | Should -Contain "n86jk8"
}
It "success download filter" {
$filt_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_download_filter.txt)
$filt_success | Should -HaveCount 3
$filt_success | Should -Contain "nxuxjy"
}
It "success already exists" {
$exist_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_already_exists.txt)
$exist_success | Should -HaveCount 3
$exist_success | Should -Contain "nxrq9g"
}
It "success hard link" {
$link_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_hard_link.txt)
$link_success | Should -HaveCount 1
$link_success | Should -Contain "nwnp2n"
}
It "success score filter" {
$score_success = (..\extract_successful_ids.ps1 example_logfiles\succeed_score_filter.txt)
$score_success | Should -HaveCount 2
$score_success | Should -Contain "ljyz27"
}
}

7
scripts/tests/test_extract_failed_ids.bats
View file

@ -7,11 +7,16 @@ teardown() {
rm -f failed.txt
}
@test "fail run no logfile" {
@test "fail run no args" {
run ../extract_failed_ids.sh
assert_failure
}
@test "fail run no logfile" {
run ../extract_failed_ids.sh ./missing.txt
assert_failure
}
@test "fail no downloader module" {
run ../extract_failed_ids.sh ./example_logfiles/failed_no_downloader.txt
echo "$output" > failed.txt

10
scripts/tests/test_extract_successful_ids.bats
View file

@ -7,6 +7,16 @@ teardown() {
rm -f successful.txt
}
@test "fail run no args" {
run ../extract_successful_ids.sh
assert_failure
}
@test "fail run no logfile" {
run ../extract_successful_ids.sh ./missing.txt
assert_failure
}
@test "success downloaded submission" {
run ../extract_successful_ids.sh ./example_logfiles/succeed_downloaded_submission.txt
echo "$output" > successful.txt