Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Imports System.Net
- Imports System.Text.RegularExpressions
- Imports System.Web
- Module modNewForum
- Public Structure NewForumPostData
- Dim Username As String
- Dim Message As String
- Dim PostNo As Integer
- Dim PostDate As String
- End Structure
- Public Structure ThreadTitleAndID
- Dim Author As String
- Dim LastReplyName As String
- Dim LastReplyTime As DateTime
- Dim Title As String
- Dim ID As Integer
- Dim RawPost As String
- Dim URL As String
- Dim TimeCreated As DateTime
- Dim Locked As Boolean
- Dim PostIt As Boolean
- Dim Poll As Boolean
- Dim NoMessages As Integer
- Dim Messages As List(Of NewForumPostData)
- End Structure
- Public Function GetNewForumLastPage(ByVal intT As Integer) As Integer
- Dim intLastMapPage As Integer = 1
- Try
- Dim wbClient As New WebClient
- Dim strPage As String = wbClient.DownloadString("https://atelier801.com/topic?f=6&t=" & intT)
- If Regex.IsMatch(strPage, Regex.Escape("<a class=""btn btn-inverse"" href=""#"" active") & ".*?>" & "(?<pages>.+?)" & "<", RegexOptions.Multiline And RegexOptions.IgnoreCase) Then
- Dim matchPages As Match = Regex.Match(strPage, Regex.Escape("<a class=""btn btn-inverse"" href=""#"" active") & ".*?>" & "(?<pages>.+?)" & "<", RegexOptions.Multiline And RegexOptions.IgnoreCase)
- intLastMapPage = matchPages.Groups("pages").Value.Split("/")(1).Trim
- Else
- intLastMapPage = 1
- End If
- Catch ex As Exception
- intLastMapPage = 1
- End Try
- Return intLastMapPage
- End Function
- Public Function GetNewFormPosts(ByVal intT As Integer, ByVal intP As Integer) As List(Of NewForumPostData)
- Dim lstNewForumPosts As New List(Of NewForumPostData)
- Dim wbClient As New WebClient
- wbClient.Encoding = System.Text.UTF8Encoding.UTF8
- Dim strPage As String = wbClient.DownloadString("https://atelier801.com/topic?f=6&t=" & intT & "&p=" & intP)
- 'https://atelier801.com/topic?f=6&t=48020&p=1
- strPage = strPage.Replace(vbLf, "").Replace(vbCr, "")
- Dim matchNames As MatchCollection = Regex.Matches(strPage, "<span class=""date-ms hidden"" data-afficher-secondes=""false"">" & "(?<postdate>.+?)" & "</span>" & _
- ".*?" & "href=""profile\?pr=" & "(?<user>.+?)" & """>" & _
- ".*?" & "<a class=""numero-message""" & ".*?" & """>" & "(?<postno>.+?)" & "</a>" & _
- ".*?" & "<div class=""cadre-message-message"">" & "(?<message>.+?)" & "</div>", RegexOptions.Multiline And RegexOptions.IgnoreCase)
- 'strPage = strPage.Replace(vbLf, "").Replace(vbCr, "")
- ' Dim matchNames As MatchCollection = Regex.Matches(strPage, "<span class=""date-ms hidden"" data-afficher-secondes=""false"">" & "(?<postdate>.+?)" & "</span>" & _
- '".*?" & "href=""profile\?pr=" & "(?<user>.+?)" & """>" & _
- '".*?" & "<a class=""numero-message""" & ".*?" & """>" & "(?<postno>.+?)" & "</a>" & _
- '".*?" & "<div id=""message_\d\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?"">" & "(?<message>.+?)" & "</div>", RegexOptions.Multiline And RegexOptions.IgnoreCase)
- For Each UserName As Match In matchNames
- Dim currPost As New NewForumPostData
- currPost.Username = StrConv(UserName.Groups("user").Value.Trim, vbProperCase)
- If Regex.IsMatch(currPost.Username, "%2B", RegexOptions.IgnoreCase) Then
- currPost.Username = Regex.Replace(currPost.Username, "%2B", "", RegexOptions.IgnoreCase)
- currPost.Username = "+" & StrConv(currPost.Username, vbProperCase)
- End If
- currPost.Message = UserName.Groups("message").Value.Trim
- currPost.Message = Regex.Replace(currPost.Message, "<div id="".*?"">", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "<div>", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "<span class=""cadre-message-modere-texte"">", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "</span>", "", RegexOptions.IgnoreCase)
- currPost.PostNo = UserName.Groups("postno").Value.Trim.Replace("#", "")
- currPost.PostDate = UserName.Groups("postdate").Value.Trim
- lstNewForumPosts.Add(currPost)
- Next
- Return lstNewForumPosts
- End Function
- ' Public Function GetNewFormPosts(ByVal intT As Integer, ByVal intP As Integer) As List(Of NewForumPostData)
- ' Dim lstNewForumPosts As New List(Of NewForumPostData)
- ' Dim wbClient As New WebClient
- ' Dim strPage As String = wbClient.DownloadString("https://atelier801.com/topic?f=6&t=" & intT & "&p=" & intP)
- ' strPage = strPage.Replace(vbLf, "").Replace(vbCr, "")
- ' Dim matchNames As MatchCollection = Regex.Matches(strPage, "<span class=""date-ms hidden"" data-afficher-secondes=""false"">" & "(?<postdate>.+?)" & "</span>" & _
- '".*?" & "href=""profile\?pr=" & "(?<user>.+?)" & """>" & _
- '".*?" & "<a class=""numero-message""" & ".*?" & """>" & "(?<postno>.+?)" & "</a>" & _
- '".*?" & "<div id=""message_\d\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?\d?"">" & "(?<message>.+?)" & "</div>", RegexOptions.Multiline And RegexOptions.IgnoreCase)
- ' For Each UserName As Match In matchNames
- ' Dim currPost As New NewForumPostData
- ' currPost.Username = StrConv(UserName.Groups("user").Value.Trim, vbProperCase)
- ' If Regex.IsMatch(currPost.Username, "%2B", RegexOptions.IgnoreCase) Then
- ' currPost.Username = Regex.Replace(currPost.Username, "%2B", "+", RegexOptions.IgnoreCase)
- ' End If
- ' currPost.Message = UserName.Groups("message").Value.Trim
- ' currPost.Message = Regex.Replace(currPost.Message, "<div id="".*?"">", "", RegexOptions.IgnoreCase)
- ' currPost.Message = Regex.Replace(currPost.Message, "<div>", "", RegexOptions.IgnoreCase)
- ' currPost.Message = Regex.Replace(currPost.Message, "<span class=""cadre-message-modere-texte"">", "", RegexOptions.IgnoreCase)
- ' currPost.Message = Regex.Replace(currPost.Message, "</span>", "", RegexOptions.IgnoreCase)
- ' currPost.PostNo = UserName.Groups("postno").Value.Trim.Replace("#", "")
- ' currPost.PostDate = UserName.Groups("postdate").Value.Trim
- ' lstNewForumPosts.Add(currPost)
- ' Next
- ' Return lstNewForumPosts
- ' End Function
- Public Function FormatForumDate(ByVal strDate As String) As String
- Dim currDate As DateTime = ConvertForumTimestamp(strDate)
- Dim strOutput As String = currDate.ToString("dd/MM/yyyy HH:mm")
- Return strOutput
- End Function
- Public Function GetAtelierThreadListNew(ByVal strPage As String, Optional ByVal booIsInbox As Boolean = False) As List(Of ThreadTitleAndID)
- 'Replace multiple spaces with single spaces
- strPage = Regex.Replace(strPage, " {2,}", " ")
- Dim lstOutput As New List(Of ThreadTitleAndID)
- 'Match each thread
- Dim matchThreads As MatchCollection = Regex.Matches(strPage, "<a class=""cadre-sujet-titre lien-blanc"" href=""(?<link>.*?)""> ?(<img src=""\/img\/icones\/(?<postIt>postit)\.png"".*?\/>)? ?(<img src=""\/img\/icones\/(?<locked>cadenas)\.png"".*?\/>)? ?(<img src=""\/img\/icones\/(?<poll>sondage)\.png"".*?\/> )?(?<title>.*?)<\/a>.*?<a class=""nombre-messages.*?>(?<noMessages>\d{1,1000})<\/a>.*?<span class=""date-ms hidden"".*?>(?<timeLastReply>.*?)<\/span>.*?<a class=""lien-blanc"" href=""profile\?pr=(?<userName>.*?)"">.*?href=""(?<threadURL>topic\?f=\d\d?&t=(?<threadID>\d{1,10}))"">.*?<span.*?><img src=""img\/icones\/16\/on-offbis(?<online>\d)\.png"".*?>(?<lastReply>.*?)<\/span>.*?date-ms hidden.*?>(?<timeCreated>.*?)<\/span>", RegexOptions.IgnoreCase Or RegexOptions.Singleline Or RegexOptions.Multiline)
- For Each currThread As Match In matchThreads
- 'WriteLogBalls(currThread.Value & vbNewLine & vbNewLine & vbNewLine)
- Dim newThread As New ThreadTitleAndID
- newThread.ID = currThread.Groups("threadID").Value
- newThread.Title = HttpUtility.HtmlDecode(currThread.Groups("title").Value.Trim)
- newThread.RawPost = currThread.Value
- newThread.URL = "https://atelier801.com/" & currThread.Groups("threadURL").Value
- newThread.Author = StrConv(HttpUtility.UrlDecode(currThread.Groups("userName").Value), vbProperCase)
- newThread.TimeCreated = ConvertForumTimestamp(currThread.Groups("timeCreated").Value)
- newThread.LastReplyName = StrConv(HttpUtility.UrlDecode(currThread.Groups("lastReply").Value), vbProperCase)
- newThread.LastReplyTime = ConvertForumTimestamp(currThread.Groups("timeLastReply").Value)
- newThread.NoMessages = CInt(currThread.Groups("noMessages").Value.Trim)
- newThread.Locked = currThread.Groups("locked").Value.Length
- newThread.PostIt = currThread.Groups("postIt").Value.Length
- newThread.Poll = currThread.Groups("poll").Value.Length
- lstOutput.Add(newThread)
- 'MsgBox("ID: " & newThread.ID & vbNewLine &
- ' "Title: " & newThread.Title & vbNewLine &
- ' "URL: " & newThread.URL & vbNewLine &
- ' "Author: " & newThread.Author & vbNewLine &
- ' "Time created: " & newThread.TimeCreated.ToString("MMM d, yyyy") & vbNewLine &
- ' "Last reply time: " & newThread.LastReplyTime.ToString("MMM d, yyyy") & vbNewLine &
- ' "Last reply name: " & newThread.LastReplyName & vbNewLine &
- ' "# messages: " & newThread.NoMessages & vbNewLine &
- ' "Locked: " & newThread.Locked & vbNewLine &
- ' "PostIt: " & newThread.PostIt & vbNewLine &
- ' "Poll: " & newThread.Poll & vbNewLine)
- Next
- Return lstOutput
- End Function
- Public Function GetAtelierThreadList(ByVal strPage As String, Optional ByVal booIsInbox As Boolean = False) As List(Of ThreadTitleAndID)
- Dim lstOutput As New List(Of ThreadTitleAndID)
- strPage = strPage.Replace(vbLf, "").Replace(vbCr, "")
- 'Dim matchThreads As MatchCollection = Regex.Matches(strPage, "<a class=""cadre-sujet-titre lien-blanc"".*?<div class=""span12"">", RegexOptions.Multiline And RegexOptions.IgnoreCase)
- Dim matchThreads As MatchCollection = Regex.Matches(strPage, "<table class=""table-cadre table-cadre-centree"">.*?<\/table>", RegexOptions.Multiline Or RegexOptions.IgnoreCase)
- For Each threadToCheck As Match In matchThreads
- Dim matchTopic As String = ""
- If booIsInbox Then
- matchTopic = Regex.Match(threadToCheck.Value, "conversation.*?co=\d\d?\d?\d?\d?\d?\d?\d?\d?", RegexOptions.IgnoreCase).Value
- matchTopic = Regex.Match(matchTopic, "\d\d?\d?\d?\d?\d?\d?\d?\d?$", RegexOptions.IgnoreCase).Value
- Else
- matchTopic = Regex.Match(threadToCheck.Value, "topic.*?t=\d\d?\d?\d?\d?\d?\d?\d?\d?", RegexOptions.IgnoreCase).Value
- matchTopic = Regex.Match(matchTopic, "\d\d?\d?\d?\d?\d?\d?\d?\d?$", RegexOptions.IgnoreCase).Value
- End If
- Dim matchTitle As String = Regex.Match(threadToCheck.Value, "<a class=""cadre-sujet-titre lien-blanc" & ".*?" & "</a>", RegexOptions.IgnoreCase).Value
- matchTitle = Regex.Replace(matchTitle, "<.*?>", "", RegexOptions.IgnoreCase).Trim
- Dim matchLink As String = "https://atelier801.com/" & Regex.Match(threadToCheck.Value, "href=""" & "(?<link>.*?)" & """", RegexOptions.IgnoreCase).Groups("link").Value.Trim
- Dim matchAuthors As MatchCollection = Regex.Matches(threadToCheck.Value, "alt=""""\>" & "(?<author>.*?)" & "\</span\>", RegexOptions.IgnoreCase)
- 'Dim matchTimeCreated As String = Regex.Match(threadToCheck.Value, " le <span class=""date-ms hidden"" .*?>" & "(?<time>.*?)" & "</span>", RegexOptions.IgnoreCase).Groups("time").Value.Trim
- Dim matchTimeCreated As String = Regex.Match(threadToCheck.Value, "<span class=""date-ms hidden"" .*?>" & "(?<time>.*?)" & "</span>", RegexOptions.IgnoreCase).Groups("time").Value.Trim
- Dim matchlastPost As Match = Regex.Match(threadToCheck.Value, "<span class=""date-ms hidden"" data-afficher-secondes=""false"">" & "(?<time>.*?)" & "</span>", RegexOptions.IgnoreCase)
- Dim matchNoMessages As Match = Regex.Match(threadToCheck.Value, "\<a class=""nombre-messages .*?\>(?<nomessages>.*?)\<\/a\>", RegexOptions.IgnoreCase)
- Dim newThread As New ThreadTitleAndID
- newThread.ID = matchTopic
- newThread.Title = HttpUtility.HtmlDecode(matchTitle)
- newThread.RawPost = threadToCheck.Value
- newThread.URL = matchLink
- 'Remove link to post and page number
- newThread.URL = Regex.Replace(newThread.URL, "#m\d\d?\d?\d?\d?\d?$", "", RegexOptions.IgnoreCase)
- newThread.URL = Regex.Replace(newThread.URL, "&p=\d\d?\d?\d?\d?\d?$", "", RegexOptions.IgnoreCase)
- 'Add max posts shown to 100
- newThread.URL = newThread.URL & "&n=100"
- newThread.Author = matchAuthors(0).Groups("author").Value 'matchAuthors(1).Groups("author").Value
- newThread.TimeCreated = ConvertMinutesTimestamp(matchTimeCreated)
- newThread.LastReplyName = "" 'matchAuthors(0).Groups("author").Value
- newThread.LastReplyTime = ConvertMinutesTimestamp(matchlastPost.Groups("time").Value.Trim)
- newThread.NoMessages = CInt(matchNoMessages.Groups("nomessages").Value.Trim)
- If threadToCheck.Value.Contains("<img src=""/img/icones/cadenas.png""") Then
- newThread.Locked = True
- Else
- newThread.Locked = False
- End If
- If threadToCheck.Value.Contains("<img src=""/img/icones/postit.png""") Then
- newThread.PostIt = True
- Else
- newThread.PostIt = False
- End If
- lstOutput.Add(newThread)
- 'WriteLogBalls(newThread.ID & " : " & newThread.Title & " : " & newThread.RawPost & " : " & newThread.URL & " : " & newThread.Author & " : " & newThread.TimeCreated & " : " & newThread.LastReplyName & " : " & newThread.LastReplyTime & " : " & newThread.NoMessages)
- Next
- Return lstOutput
- End Function
- Public Function ConvertMinutesTimestamp(ByVal timestamp As String) As DateTime
- Return New DateTime(1970, 1, 1, 0, 0, 0).AddMilliseconds(timestamp).AddHours(-4)
- End Function
- Public Function GetInboxMessagesSource(ByVal strPage As String) As List(Of NewForumPostData)
- Dim lstOutput As New List(Of NewForumPostData)
- Dim matchPostsOnPage As MatchCollection = Regex.Matches(strPage, _
- "<span class=""date-ms .*?>(?<timeofpost>.*?)<\/span>.*?profile\?pr=(?<username>.*?)"">.*?<div id=""citer_message.*?>(?<message>.*?)<\/div>.*?<a class=""numero-message"".*?>(?<postno>.*?)<\/a>", RegexOptions.Singleline Or RegexOptions.IgnoreCase)
- For Each postMatch As Match In matchPostsOnPage
- Dim currPost As New NewForumPostData
- currPost.Username = StrConv(postMatch.Groups("username").Value.Trim, vbProperCase)
- If Regex.IsMatch(currPost.Username, "%2B", RegexOptions.IgnoreCase) Then
- currPost.Username = Regex.Replace(currPost.Username, "%2B", "", RegexOptions.IgnoreCase)
- currPost.Username = "+" & StrConv(currPost.Username, vbProperCase)
- End If
- currPost.Message = HttpUtility.HtmlDecode(postMatch.Groups("message").Value.Trim)
- currPost.Message = currPost.Message.Replace(vbLf, vbLf & Chr(9))
- currPost.PostNo = postMatch.Groups("postno").Value.Trim.Replace("#", "")
- currPost.PostDate = postMatch.Groups("timeofpost").Value.Trim
- lstOutput.Add(currPost)
- Next
- Return lstOutput
- End Function
- Public Function GetNewFormPostsSource(ByVal strPage As String) As List(Of NewForumPostData)
- Dim lstNewForumPosts As New List(Of NewForumPostData)
- 'Dim matchForumPost As MatchCollection = Regex.Matches(strPage, "<span class=""date-minutes hidden"">" & "(?<postdate>.+?)" & "</span>" & _
- '".*?" & "href=""profile\?pr=" & "(?<user>.+?)" & """>" & _
- '".*?" & "<a class=""numero-message""" & ".*?" & """>" & "(?<postno>.+?)" & "</a>" & _
- '".*?" & "<div class=""cadre-message-message"">" & "(?<message>.+?)" & "</div>", RegexOptions.Singleline Or RegexOptions.IgnoreCase)
- Dim matchForumPost As MatchCollection = Regex.Matches(strPage, "<span class=""date-ms hidden"" data-afficher-secondes=""false"">" & "(?<postdate>.+?)" & "</span>" & _
- ".*?" & "href=""profile\?pr=" & "(?<user>.*?)" & """>" & _
- ".*?" & "<a class=""numero-message""" & ".*?" & """>" & "(?<postno>.*?)" & "</a>" & _
- ".*?" & "<div class=""cadre-message-message"">" & "(?<message>.*?)" & "</div>", RegexOptions.Singleline Or RegexOptions.IgnoreCase)
- For Each postMatch As Match In matchForumPost
- Dim currPost As New NewForumPostData
- currPost.Username = StrConv(postMatch.Groups("user").Value.Trim, vbProperCase)
- If Regex.IsMatch(currPost.Username, "%2B", RegexOptions.IgnoreCase) Then
- currPost.Username = Regex.Replace(currPost.Username, "%2B", "", RegexOptions.IgnoreCase)
- currPost.Username = "+" & StrConv(currPost.Username, vbProperCase)
- End If
- currPost.Message = postMatch.Groups("message").Value.Trim
- currPost.Message = Regex.Replace(currPost.Message, "<div id="".*?"">", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "<div>", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "<span class=""cadre-message-modere-texte"">", "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "</span>", "", RegexOptions.IgnoreCase)
- currPost.Message = HttpUtility.HtmlDecode(currPost.Message)
- currPost.Message = Regex.Replace(currPost.Message, Regex.Escape("<span style=""font-family:Tahoma;""><span style=""font-size:10px;"">=^<span style=""color:#30BA76;"">?<span style=""font-size:10px;"">?<span style=""color:#30BA76;"">?^<span style=""font-size:10px;"">="), "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, Regex.Escape("<span style=""font-family:Tahoma;""><span style=""font-size:10px;"">=^<span style=""color:#30BA76;"">●<span style=""font-size:10px;"">⋏<span style=""color:#30BA76;"">●^<span style=""font-size:10px;"">="), "", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, "<br ?/?>", " ", RegexOptions.IgnoreCase)
- Dim matchLinks As MatchCollection = Regex.Matches(currPost.Message, "<a href.*?</a>", RegexOptions.IgnoreCase)
- For Each link As Match In matchLinks
- Dim matchCurrLink As Match = Regex.Match(link.Value, "<a href=""(?<url>.*?)"".*?>(?<text>.*?)</a>", RegexOptions.IgnoreCase)
- currPost.Message = Regex.Replace(currPost.Message, Regex.Escape(matchCurrLink.Value), matchCurrLink.Groups("text").Value & ": " & matchCurrLink.Groups("url").Value, RegexOptions.IgnoreCase)
- Next
- currPost.PostNo = postMatch.Groups("postno").Value.Trim.Replace("#", "")
- currPost.PostDate = postMatch.Groups("postdate").Value.Trim
- lstNewForumPosts.Add(currPost)
- Next
- Return lstNewForumPosts
- End Function
- Sub WriteLogBalls(ByVal strLine As String)
- Using swWriteLog As New System.IO.StreamWriter("C:\Users\inkan\Dropbox\new\Mewbzz\0_log.txt", True)
- swWriteLog.WriteLine(strLine)
- End Using
- End Sub
- Sub WriteLogBalls2(ByVal strLine As String)
- Using swWriteLog As New System.IO.StreamWriter("C:\Users\inkan\Dropbox\new\Mewbzz\0000_log.txt", True)
- swWriteLog.WriteLine(strLine)
- End Using
- End Sub
- Public Sub DeleteLogBalls()
- If System.IO.File.Exists("C:\Users\inkan\Dropbox\new\Mewbzz\0_log.txt") Then
- System.IO.File.Delete("C:\Users\inkan\Dropbox\new\Mewbzz\0_log.txt")
- End If
- End Sub
- End Module
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement