MUN

package main

import (
  "fmt"
  "log"
  "strconv"
  "strings"
  "regexp"
  "os"
  "github.com/PuerkitoBio/goquery"
  "github.com/tealeg/xlsx"
)

var xls_file *xlsx.File
var sheet *xlsx.Sheet
var row *xlsx.Row
var cell *xlsx.Cell
var maxPhones = 20
var maxEmails = 20

func validateEmail(email string) bool {
 Re := regexp.MustCompile(`^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,4}$`)
 return Re.MatchString(email)
}

func Scrape() {

  emails_processed := make(map[string]string)

  phones_temp := map[int]string {
    0: "-",
    1: "-",
    2: "-",
    3: "-",
    4: "-",
    5: "-",
    6: "-",
    7: "-",
    8: "-",
    9: "-",
    10: "-",
    11: "-",
    12: "-",
    13: "-",
    14: "-",
    15: "-",
    16: "-",
    17: "-",
    18: "-",
    19: "-",
  }

  emails_temp := map[int]string {
    0: "-",
    1: "-",
    2: "-",
    3: "-",
    4: "-",
    5: "-",
    6: "-",
    7: "-",
    8: "-",
    9: "-",
    10: "-",
    11: "-",
    12: "-",
    13: "-",
    14: "-",
    15: "-",
    16: "-",
    17: "-",
    18: "-",
    19: "-",
  }

  doc, err := goquery.NewDocument("http://www.mininterior.gov.ar/municipios/datos-municipio.php")

  if err != nil {
    log.Fatal(err)
  }

  // open files r and w
  file, err := os.OpenFile("emails.txt", os.O_APPEND|os.O_WRONLY|os.O_CREATE|os.O_TRUNC,0600)

  if err != nil {
      panic(err)
  }
  defer file.Close()

  xls_file = xlsx.NewFile()
  sheet, err = xls_file.AddSheet("Sheet1")
  if err != nil {
      fmt.Printf(err.Error())
  }
  row = sheet.AddRow()
  cell = row.AddCell()
  cell.Value = "Código"
  cell = row.AddCell()
  cell.Value = "Provincia"
  cell = row.AddCell()
  cell.Value = "Municipio"
  cell = row.AddCell()
  cell.Value = "Sitio Web"
  cell = row.AddCell()
  cell.Value = "Dirección"
  for i:=1; i <= maxPhones; i++ {
    cell = row.AddCell()
    cell.Value = "Teléfono " + strconv.Itoa(i)
  }
  for i:=1; i <= maxEmails; i++ {
    cell = row.AddCell()
    cell.Value = "Email " + strconv.Itoa(i)
  }

  doc.Find("#provincia option").Each(func(i int, s *goquery.Selection) {

    provPrefix, _ := s.Attr("value")
    provName := s.Text()

    if provPrefix != "" {

      currItem := 1

      for currItem >= 1 {

        numToStr := strconv.Itoa(currItem)

        if len(numToStr) == 1 {
          numToStr = "00" + numToStr
        } else if len(numToStr) == 2 {
          numToStr = "0" + numToStr
        }

        doc, err := goquery.NewDocument("http://www.mininterior.gov.ar/municipios/masinfo.php?municipio=" + provPrefix + numToStr)

        if err != nil {
          log.Fatal(err)
        }

        muni_name := doc.Find("h1").Eq(1).Text()
        muni_web := doc.Find("table").Eq(2).Find("td").Eq(7).Text()
        muni_addr := doc.Find("table").Eq(2).Find("td").Eq(1).Text()
        phones, _ := doc.Find("table").Eq(2).Find("td").Eq(3).Html()
        emails, _ := doc.Find("table").Eq(2).Find("td").Eq(5).Html()

        if muni_name == "" {

          currItem = 1
          break

        } else {

          phones_temp = map[int]string {
            0: "-",
            1: "-",
            2: "-",
            3: "-",
            4: "-",
            5: "-",
            6: "-",
            7: "-",
            8: "-",
            9: "-",
            10: "-",
            11: "-",
            12: "-",
            13: "-",
            14: "-",
            15: "-",
            16: "-",
            17: "-",
            18: "-",
            19: "-",
          }

          emails_temp = map[int]string {
            0: "-",
            1: "-",
            2: "-",
            3: "-",
            4: "-",
            5: "-",
            6: "-",
            7: "-",
            8: "-",
            9: "-",
            10: "-",
            11: "-",
            12: "-",
            13: "-",
            14: "-",
            15: "-",
            16: "-",
            17: "-",
            18: "-",
            19: "-",
          }

          row = sheet.AddRow()

          cell = row.AddCell()
          cell.Value = provPrefix
          cell = row.AddCell()
          cell.Value = provName
          cell = row.AddCell()
          cell.Value = muni_name
          cell = row.AddCell()
          cell.Value = muni_web
          cell = row.AddCell()
          cell.Value = muni_addr

          phones_temp_existing := strings.Split(phones, "<br/>")
          emails_temp_existing := strings.Split(emails, "<br/>")

          x := 0

          for i := range phones_temp_existing {

            phones_temp_existing[i] = strings.Replace(phones_temp_existing[i], "<strong>", "", -1)
            phones_temp_existing[i] = strings.Replace(phones_temp_existing[i], "</strong>", "", -1)
            phones_temp_existing[i] = strings.Replace(phones_temp_existing[i], "<p>", "", -1)
            phones_temp_existing[i] = strings.Replace(phones_temp_existing[i], "</p>", "", -1)
            if phones_temp_existing[i] != "" && phones_temp_existing[i] != " " && phones_temp_existing[i] != "-" && phones_temp_existing[i] != "<strong>" && phones_temp_existing[i] != "</strong>" {
              phones_temp[x] = phones_temp_existing[i]
              x++
            }

          }

          x = 0

          for i := range emails_temp_existing {
            emails_temp_existing[i] = strings.Replace(emails_temp_existing[i], "<strong>", "", -1)
            emails_temp_existing[i] = strings.Replace(emails_temp_existing[i], "</strong>", "", -1)
            emails_temp_existing[i] = strings.Replace(emails_temp_existing[i], "<p>", "", -1)
            emails_temp_existing[i] = strings.Replace(emails_temp_existing[i], "</p>", "", -1)
            if emails_temp_existing[i] != "" && emails_temp_existing[i] != " " && emails_temp_existing[i] != "-" && emails_temp_existing[i] != "<strong>" && emails_temp_existing[i] != "</strong>" && validateEmail(emails_temp_existing[i]) {
              emails_temp[x] = emails_temp_existing[i]
              x++
            }
          }

          for i := range phones_temp {

            cell = row.AddCell()
            cell.Value = phones_temp[i]

          }

          for i := range emails_temp {

            _, ok := emails_processed[emails_temp[i]]

            if !ok && emails_temp[i] != "-" {

              fmt.Println(provPrefix + numToStr + " => " + emails_temp[i])

              emails_processed[emails_temp[i]] = emails_temp[i]

              if _, err = file.WriteString(muni_name + "||" + emails_temp[i] + "\r\n"); err != nil {
               panic(err)
              }

            }

            cell = row.AddCell()
            cell.Value = emails_temp[i]

          }

        }

        currItem = currItem + 1

      }
    }
  })

  err = xls_file.Save("municipios.xlsx")
  if err != nil {
      fmt.Printf(err.Error())
  }

}

func main() {
  Scrape()
}