API tools faq
paste
fant0men

Crawl Be software archive

fant0men
Dec 6th, 2020 (edited)
321
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.74 KB | None | 0 0
raw download clone embed print report
  1. regex='href=\"(\.\.\/)*([^ "]+)'
  2. agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36'
  3.  
  4. curl --user-agent "$agent" --retry 10 --retry-delay 10 --connect-timeout 10 --silent "http://testou.free.fr/www.beatjapan.org/mirror/www.be.com/beware/Available_List_Source.html" | grep -Eo "$regex" | sed -E "s/${regex}/\2/g" | while read line; do wget -e robots=off --waitretry=10 --tries=inf "http://testou.free.fr/${line}"; done
  5.  
  6. ***
  7.  
  8. find . -type f -iname "*" | sed 's/^\.\///' | while read line; do f_type=$(file "$line" | grep -Fq ': HTML document,'; echo "$?"); if [[ $f_type -eq 0 ]]; then echo "$line"; fi; done
  9.  
  10. ***
  11.  
  12. for zip in '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 5.0 Professional.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 5.0 Personal.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 4.5.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 4.0.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 3.1.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS 3.0.7z' '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BeOS/Distr/BeOS Developer Edition 1.1.7z'; do bn=$(basename "$zip" | sed 's/\.7z$//'); if [[ ! -d $bn ]]; then mkdir -p "$bn" || continue; echo "$bn"; cd "$bn"; arch_unpack "$zip"; cd ..; fi; done
  13.  
  14. ***
  15.  
  16. ***
  17.  
  18. remove haiku images
  19. add zeta images
  20. remove duplicates
  21.  
  22. extract sit file
  23. sort docs
  24. sort floppy images / updates
  25.  
  26. merge development/docs
  27. file:///run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BEOS5PEWARE/development/docs
  28.  
  29. 'documentation'
  30.  
  31. smc?
  32.  
  33. * sort by largest 7z and find more install images
  34.  
  35. * update hqx_backup, sit_backup
  36. * find and remove empty dirs (hqx_backup, sit_backup)
  37.  
  38. * remove '_zzz$'
  39.  
  40. ***
  41.  
  42. wget -r -np -e robots=off --waitretry=10 --tries=inf http://ftp.icm.edu.pl/pub/beos/
  43.  
  44. wget -r -np -k -e robots=off --waitretry=10 --tries=inf "https://birdhouse.org/beos/bible/"
  45.  
  46. ***
  47.  
  48. find . -type f -iname "*.tar" | while read f; do arch_pack "${f}.7z" "$f" && rm "$f"; done
  49. find . -type f -iname "*.pkg" | while read f; do arch_pack "${f}.7z" "$f" && rm "$f"; done
  50.  
  51. find . -type f -iname "*.tar" | while read f; do if [[ ! -f ${f}.7z ]]; then of="${f}.7z"; else of="${f%.tar}-${RANDOM}.tar.7z"; fi; arch_pack "$of" "$f" && rm "$f"; done
  52. find . -type f -iname "*.pkg" | while read f; do if [[ ! -f ${f}.7z ]]; then of="${f}.7z"; else of="${f%.pkg}-${RANDOM}.pkg.7z"; fi; arch_pack "$of" "$f" && rm "$f"; done
  53.  
  54. ***
  55.  
  56. find '/run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection' -type f -iname "*.pdf" | while read f; do newfn=$(sed -E 's/pdf$/pdf/i' <<<"$f"); if [[ $f != $newfn ]]; then mv -n "$f" "$newfn"; fi; done
  57.  
  58. ***
  59.  
  60. find . -type f -iname "*.7z" | while read f; do 7za l "$f" | tail -n 1 | grep -Eq "[[:blank:]]+0[[:blank:]]+files" && echo "$f" >> /home/lucifer/beos_empty_7z.txt; done
  61.  
  62. ***
  63.  
  64. ipro1000_0.4.zip
  65.  
  66. ***
  67.  
  68. find . -type f -iname "index.html\?*" | while read f; do rm "$f"; done
  69.  
  70. ***
  71.  
  72. find . -type f -iname "*.gz" -o -iname "*.bz2" | grep -Ev '.tar.(gz|bz2)$'
  73.  
  74. ***
  75.  
  76. find . -type f -iname "*.zip.bak" | while read f; do rm "$f"; done
  77.  
  78. find . -type d | grep -E '\-[[:digit:]]+$'
  79.  
  80. ***
  81.  
  82. find corrupted archives:
  83. cat md5.db | while read line; do md5=$(grep -Eo '[[:alnum:]]{32}$' <<<"$line"); fn=$(grep "$md5" '/home/lucifer/hqx_backup_old/md5.db' | cut -d'*' -f1 | sed 's/[[:blank:]]+$//'); echo "*** ${fn}"; done
  84.  
  85. grep '\.hqx' md5.db | while read line; do md5=$(grep -Eo '[[:alnum:]]{32}$' <<<"$line"); fn=$(grep "$md5" '/home/lucifer/hqx_backup_old/md5.db' | cut -d'*' -f1 | sed 's/[[:blank:]]+$//'); echo "*** ${fn}"; done
  86.  
  87. find non-zip files, and gzip archives that are not actually gzip:
  88. find . -type f | grep -Eiv '\.zip$' | while read f; do file "$f" | grep -Eiv "gzip compressed data"; done
  89.  
  90. ***
  91.  
  92. sed -E 's/([^ a-zA-Z0-9\.\-_ ])/\\\1/g'
  93.  
  94. ***
  95.  
  96. find . -type f -iname "*.img" | while read f; do arch_pack "${f}.7z" "$f" && rm "$f"; done
  97.  
  98. ***
  99.  
  100. find . -type f -iname "*.sea" -o -iname "*.rsrc" | while read f; do fn=$(sed -E 's/^\.\///' <<<"$f"); dn=$(dirname "$fn"); mkdir -p "sea/${dn}"; mv -n "$fn" "sea/${dn}"; done
  101.  
  102. ***
  103.  
  104. find . -type f -iname "*.gif" | while read f; do newfn=$(sed -E 's/\.gif$/\.gif/i' <<<"$f"); mv -n "$f" "$newfn"; done
  105.  
  106. ***
  107.  
  108. regex='((\.lzh)|(\.gz)|(\.z)|(\.bz2)|(\.xz)|(\.zip)|(\.rar)|(\.arj)|(\.tar)|(\.tgz)|(\.tbz)|(\.tbz2)|(\.txz)|(\.sit)|(\.hqx)|(\.sea)|(\.rsrc))$'
  109.  
  110. ***
  111.  
  112. find . -type f -iname "*be*book*" -o -iname "*bible*"
  113.  
  114.  
  115.  
  116. /run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BEOS5PEWARE/system_files/icons/CubeBeOSIcons1.7z
  117. /run/media/lucifer/f61840d6-9ba6-4cf8-ad6f-5c97c8c58b18/Torrents/BeOS collection/BEOS5PEWARE/system_files/icons/CLBeIcons1.7z
  118.  
  119. ***
  120.  
  121. check_beos.sh (check if out dir is empty)
  122.  
  123. ***
  124.  
  125. for zip in *.7z; do mkdir "${zip%.7z}"; cd "${zip%.7z}"; arch_unpack ../"$zip"; cd ..; done
  126.  
  127. ***
  128.  
  129. find . -type f | grep -Ev '\.pkg.7z$' | while read f; do rm "$f"; done
  130.  
  131. ***
  132.  
  133. tr -d '\r' <'md5.db' | while read line; do fn=$(sed -E 's/^(.*)\t\*\t[[:alnum:]]{32}/\1/' <<<"$line"); if [[ -f $fn ]]; then echo "$line"; fi; done
  134.  
  135. ***
  136.  
  137. find . -type f -iname "*" | sed -E 's/\.[[:alnum:]]{1,3}$//' | sort -u | while read fn; do files=("${fn}"*); arch_pack "${fn}.7z" "${files[@]}"; rm "${files[@]}"; done
  138.  
  139. ***
  140.  
  141. grep -E -e '\.bin$' -e '\.cue$' -e '\.iso$' -e '\.7z$' -e '\.pdf$' -e '\.exe$' beos_duplicates.txt | sort
  142.  
  143. ***
  144.  
  145. find . -iname "*.hqx*" | sed -E 's/^\.\///' | while read f; do dn=$(dirname "$f"); mkdir -p "/home/lucifer/hqx_backup/${dn}"; mv -n "$f" "/home/lucifer/hqx_backup/${dn}"; done
  146. find . -iname "*.sea*" | sed -E 's/^\.\///' | while read f; do dn=$(dirname "$f"); mkdir -p "/home/lucifer/sit_backup/${dn}"; mv -n "$f" "/home/lucifer/sit_backup/${dn}"; done
  147.  
  148. ***
  149.  
  150. for dir in *; do find "$dir" -type f -iname "*"; while read f; do mv -n "$f" "$dir"; done; done
  151.  
  152. ***
  153.  
  154. find . -type f -iname "*.pdf" | while read f; do arch_pack "${f%.pdf}.7z" "$f"; rm "$f"; done
  155.  
  156. ***
  157.  
  158. find . -type f -iname "index.html" | while read f; do grep -Fq 'Directory Listing' "$f" && rm "$f"; done
  159. find . -type f -iname "index.html" | while read f; do grep -Fq 'Request_URI' "$f" && rm "$f"; done
  160.  
  161. ***
  162.  
  163. find . -type f -iname "*bin cue*.7z" | while read f; do dn=$(dirname "$f"); bn=$(basename "$f"); cd "$dn"; mkdir "${bn%.7z}"; cd "${bn%.7z}"; arch_unpack ../"$bn"; cd ../..; done
  164.  
  165. ***
  166.  
  167. find . -iname "index.html" | while read f; do grep -Fq '>Index of ' "$f" && echo "$f"; done
  168.  
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!