Get-WebDirectory.ps1

[CmdletBinding()]
param(
    [Parameter(Mandatory=$true)][string]$url,
    [Parameter()][string]$dest='.\',
    [Parameter()][switch]$UseBITS=$true,
    [Parameter()][switch]$Recurse=$true,
    [Parameter()][string[]]$FileTypes,
    [Parameter()][switch]$ListOnly=$false,
    [Parameter()][switch]$RecurseIntoFolders=$true,
    [Parameter()][int16]$Wait=0,
    [Parameter()][System.Management.Automation.PSCredential]$Credential
    )

Function GetDir($folder, $CurrentUrl, $destfldr) {
    if(!$CurrentUrl.EndsWith('/')) {$CurrentUrl += '/'}
    $fname = $null
    $href = $null
    $head = $null
    ForEach($link in $folder.links) {
        $fname = $link.innerHtml
        $href = $link.href
        If (@('Name','Size','Last modified','Description','Parent Directory','../',$hostname, '[To Parent Directory]', 'Parent directory/', '#', 'File Name', 'File Size', ' ↓ ', 'Date') -notcontains $fname -and $href -notlike '`?*') {
            $head = Invoke-WebRequest $CurrentUrl$href -Method Head -Credential $Credential
            If($head.Headers['content-type'] -like 'text/html*') {
                #Assume folder
                If(!$href.EndsWith('/')) { $href += '/'}
                if($Recurse) {
                    Write-Host "Navigating to subfolder $CurrentUrl$href"
                    Try {
                        $subfolder = Invoke-WebRequest $CurrentUrl$href -Credential $Credential

                        If($RecurseIntoFolders) {
                            $destfldr = $dest + $href.Replace('/', '\')
                        }
                        GetDir -Folder $subfolder -CurrentUrl $CurrentUrl$href -destfldr $destfldr
                    }
                    catch { Write-Host "Error reading $CurrentUrl$href $($_.Exception.Message) line $($_.InvocationInfo.ScriptLineNumber)" -ForegroundColor Red }
                }
            } else {
                if(@('Name','Size','Last modified','Description','Parent Directory','../',$hostname, '[To Parent Directory]', '#') -notcontains $href `
                -and ($FileTypes -eq $null -or $FileTypes -contains [System.IO.Path]::GetExtension($href))) {
                    $htmlfile = $href.Replace('&','&')
                    $file = [System.IO.Path]::GetFileName([System.Net.WebUtility]::UrlDecode($href))
                    if(!(Test-Path -LiteralPath $destfldr$file)) {
                        if($ListOnly) {
                            "$CurrentUrl$file" | Out-File $outfile -Append
                        } else {
                            Write-Host "Getting $CurrentUrl$file"
                            #pause
                            $dir = [System.IO.Path]::GetDirectoryName("$destfldr$file")
                            if(!(Test-Path $dir)) {
                                New-Item $dir -ItemType Directory -Force
                            }
                            if($UseBITS) {
                                Start-BitsTransfer -Source $CurrentUrl$htmlfile -Destination $destfldr$file
                            } else {
                                Invoke-WebRequest $CurrentUrl$htmlfile -outfile $destfldr$file -ErrorAction SilentlyContinue -Credential $Credential
                            }
                            if($wait -gt 0) {Start-Sleep $wait}
                        }
                    }
                }
            }
        }
    }
}

$start = Get-Date
Import-Module BitsTransfer

$hostname = ([System.Uri]$url).Host

if($ListOnly) {
    $outfile = "$dest$hostname.lst"
    "#Directory listing of $($hostname):" | Out-File $outfile
}

if($dest.Substring($dest.Length-1, 1) -ne '\') { $dest +='\' }
if(!(Test-Path $dest)) { New-Item $dest -type directory }

if($FileTypes -ne $null) {
    For($i=0; $i -lt $FileTypes.length; $i++) {
        $ext = $FileTypes[$i]
        if (!$ext.StartsWith('.')) {
            $ext = ".$ext"
            $FileTypes[$i] = $ext
        }
    }
}

$folder = Invoke-WebRequest $url -Credential $Credential

GetDir -Folder $folder -CurrentUrl $url -destfldr $dest

[gc]::collect()
$end = Get-Date
$elapsed = $end - $start

Write-Host "Done! $elapsed" -ForegroundColor Green