proc handle_bgexec {callback chan} {
  append ::bgexec_data($chan) [read $chan]
  if {[eof $chan]} {
    {*}$callback $::bgexec_data($chan)
    close $chan
    unset ::bgexec_data($chan)
  }
}

proc bgexec {callback args} {
  set chan [open "| $args" r]
  fconfigure $chan -blocking false
  fileevent $chan readable [list handle_bgexec $callback $chan]
  return
}
set U 0
proc job_done {url result} {
  incr ::U
  if {$result eq "-1" || $result eq "null"} {
      puts "Error: $url"
  } else {
    #puts $result
        puts "OK -- $url"
    app "$result\n"
  }
  puts "\#$::U"
  set ::LAST($url) 1
}
console show

proc app {txt} {
    set fp [open "products.txt" a+]
    puts $fp $txt
  puts $fp "~~~~~"
    close $fp
}

proc generateExport {} {
    set header {id;Title;_vc_post_settings;pp_sidebar_layout;pp_woo_thumbnail_style;slide_template;_thumbnail_id;pp_parallax_opacity;pp_woo_parallax_type;_wc_rating_count;_wc_review_count;_wc_average_rating;desc;desc-short;image;price;second_price;categ}
    set lineOut {ID;TITLE;"a:1:{s:10:""vc_grid_id"";a:0:{}}";full-width;vertical;default;2624;0;default;a:0:{};0;0;DESC;DESCSHORT;IMG;PRICE;;CATEG }"


    set incom [open "1.txt" r]
    set ::exportFile "exp.csv"

    set fp [open $::exportFile w+]
    fconfigure $fp

    puts $fp $header
    set id 1

    while {[gets $incom line] != -1} {
        regsub -all -- " " $line " " line
        regsub -all -- ";" $line "***" line
        if {[string match {PRICE:*} $line]} {
            regexp {PRICE:(.*)} $line => price
            regsub "PRICE" $lineOut $price lineOut
        }
        if {[string match {TITLE:*} $line]} {
            regexp {TITLE:(.*)} $line => title
            regsub {\([0-9A-Z]{8}\)} $title "" title
            regsub "TITLE" $lineOut $title lineOut
        }
        if {[string match {IMG:*} $line]} {
            regexp {IMG:(.*)} $line => img
            regsub "IMG" $lineOut $img lineOut
        }
        if {[string match {DESC:*} $line]} {
            regexp {DESC:(.*)} $line => desc
            regsub "DESC" $lineOut $desc lineOut
        }
        if {[string match {DESCSHORT:*} $line]} {
            regexp {DESCSHORT:(.*)} $line => desc_short
            regsub "DESCSHORT" $lineOut $desc_short lineOut
        }
        if {[string match {CATEG:*} $line]} {
            regexp {CATEG:(.*)} $line => ct
            regsub "CATEG" $lineOut $ct lineOut
        }

        if {$line == "~~~~~"} {
            regsub "ID" $lineOut $id lineOut
            incr id
            #regsub -all -- "%br%" $lineOut "\r\n" lineOut
            puts $fp $lineOut
            set lineOut {ID;TITLE;"a:1:{s:10:""vc_grid_id"";a:0:{}}";full-width;vertical;default;2624;0;default;a:0:{};0;0;DESC;DESCSHORT;IMG;PRICE;;CATEG }
        }
    }

    close $fp
    close $incom
}




if {[file exist "products.txt"]} {file delete "products.txt"}

set fp [open "urls.txt" r]
set urls [read $fp]
close $fp
set ::lastCat {!!!!!}

foreach url [split $urls "\n"] {
  if {[string length [string trim $url]] < 3} continue;
  if {[string index $url 0] eq "!"} {
    set ::lastCat [string range $url 1 end]
    continue
  }
  set ::LAST($url) 0
  puts $url
  if {[catch {bgexec "job_done $url" phantomjs.exe --output-encoding=cp1251 --disk-cache=true "parser.js" $url "$::lastCat"}]} {
    puts "ERROR $url"
  }
  vwait ::LAST($url)
  update idletasks
}

set incom [open "products.txt" r]
set fp [open 1.txt w+]
fconfigure $fp -encoding utf-8
while {[gets $incom line] != -1} {
        if {[string match {PRICE:*} $line]} {
            puts $fp $line
    } elseif {[string match {TITLE:*} $line]} {
        puts $fp $line
    } elseif {[string match {IMG:*} $line]} {
        puts $fp $line
    } elseif {[string match {CATEG:*} $line]} {
        puts $fp $line
    } elseif {[string match {DESC:*} $line]} {
        puts $fp $line
    } elseif {[string match {DESCSHORT:*} $line]} {
      puts $fp $line
    } elseif {$line == "~~~~~"} {
        puts $fp "~~~~~\n"
    }
}
close $fp

generateExport