__author__ = 'DarkWeb' # Here, we are importing the auxiliary functions to clean or convert data import datetime from Forums.Utilities.utilities import * from datetime import date from datetime import timedelta import re import traceback # Here, we are importing BeautifulSoup to search through the HTML tree from bs4 import BeautifulSoup # This is the method to parse the Description Pages (one page to each topic in the Listing Pages) def dread_description_parser(soup): # Fields to be parsed topic = "-1" # topic name user = [] # all users of each post addDate = [] # all dated of each post feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) status = [] # all user's authority in each post such as (adm, member, dangerous) reputation = [] # all users's karma in each post (usually found as a number) sign = [] # all user's signature in each post (usually a standard message after the content of the post) post = [] # all messages of each post interest = [] # all user's interest in each post # Finding the topic (should be just one coming from the Listing Page) container = soup.find('div', {"class": "content"}) li = container.find("a", {"class": "title"}) if li == None: return None topic = li.text topic = topic.replace(u'\xa0', ' ') topic = topic.replace(",","") topic = topic.replace("\n","") topic = cleanString(topic.strip()) # print(topic) # Finding the repeated tag that corresponds to the listing of posts # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \ # soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"}) try: # putting the initial post data since it is separated from comments # author name init_post = container.find('div', {"class": "item"}) author = init_post.find('div', {"class": "author"}).select_one('a[href^="/u/"]').text flair = init_post.find('div', {"class": "author"}).find("span", {"class": "flair"}) try: flair = flair.text.strip() author = author.replace(flair, '') except: pass author = author.strip() user.append(cleanString(author)) # status flair = init_post.find("span", {"class": "flair"}) if flair != None: flair = flair.text.strip() else: flair = "-1" status.append(cleanString(flair)) # no blurb interest.append(-1) # points for post karma = init_post.find("div", {"class": "voteCount"}) if karma != None: karma = karma.text karma = karma.replace("points", "") karma = karma.replace(":", "") karma = karma.strip() else: karma = "-1" reputation.append(cleanString(karma)) # date spans = init_post.find('div', {"class": "author"}).find('span', recursive=False) dt = spans['title'] month = find_month(dt) split_text = dt.split() day = int(re.search(r'\d+', split_text[0]).group()) year = int(split_text[2]) hm = re.findall(r'\d+', split_text[-1]) hm[0] = int(hm[0]) hm[1] = int(hm[1]) date_time_obj = datetime(year, month, day, hour=hm[0], minute=hm[1]) addDate.append(date_time_obj) # content inner = init_post.find("div", {"class": "postContent"}) inner = inner.text.strip() post.append(cleanString(inner)) # no signature sign.append(-1) # no feedback feedback.append(-1) comments = soup.find('div', {"class": "postComments"}) if comments == None: row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) return row else: comments = soup.find('div', {"class": "postComments"}).find_all('div', "comment") # print(len(posts)) # For each message (post), get all the fields we are interested to: for ipost in comments: # Finding a first level of the HTML page # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"}) cc = ipost.find('div', {"class": "commentContent"}) post_wrapper = cc.find('a', {"class": "username"}).text flair = cc.find("span", {"class": "flair"}) try: flair = flair.text.strip() post_wrapper = post_wrapper.replace(flair, '') except: pass author = post_wrapper.strip() user.append(cleanString(author)) # Finding the status of the author # Dread does not have membergroup and postgroup, but it has flair, similar enough postgroup = None if flair != None: if postgroup != None: postgroup = postgroup.text.strip() flair = flair + " - " + postgroup else: if postgroup != None: flair = postgroup.text.strip() else: flair = "-1" status.append(cleanString(flair)) # print("status " + cleanString(membergroup)) # Finding the interest of the author # Dread does not have blurb interest.append(-1) # Finding the reputation of the user # Dread doesn't have reputation per user, but instead each post has its own point system karma = cc.find('div', {"class": "votes"}) if karma != None: karma = karma.text karma = karma.replace("points","") karma = karma.replace(":", "") karma = karma.strip() else: karma = "-1" reputation.append(cleanString(karma)) # print("karma " + cleanString(karma)) # Getting here another good tag to find the post date, post content and users' signature postarea = ipost.find('div', {"class": "timestamp"}).find('span', recursive=False) dt = postarea['title'] month = find_month(dt) split_text = dt.split() day = int(re.search(r'\d+', split_text[0]).group()) year = int(split_text[2]) hm = re.findall(r'\d+', split_text[-1]) hm[0] = int(hm[0]) hm[1] = int(hm[1]) date_time_obj = datetime(year, month, day, hour=hm[0], minute=hm[1]) addDate.append(date_time_obj) # Finding the post inner = ipost.find('div', {"class": "commentBody"}) inner = inner.text.strip() # print(inner) post.append(cleanString(inner)) # No signature for Dread sign.append(-1) # As no information about users's feedback was found, just assign "-1" to the variable feedback.append("-1") except : traceback.print_exc() # if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": # user.append("-1") # status.append(-1) # interest.append(-1) # reputation.append(-1) # addDate.append(-1) # post.append("NO ACCESS TO THIS PAGE!") # sign.append(-1) # feedback.append(-1) # Populate the final variable (this should be a list with all fields scraped) row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) # Sending the results return row # This is the method to parse the Listing Pages (one page with many posts) def dread_listing_parser(soup): board = "-1" # board name (the previous level of the topic in the Forum categorization tree. # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) nm = 0 # this variable should receive the number of topics topic = [] # all topics user = [] # all users of each topic post = [] # number of posts of each topic view = [] # number of views of each topic addDate = [] # when the topic was created (difficult to find) href = [] # this variable should receive all cleaned urls (we will use this to do the marge between # Listing and Description pages) # Finding the board (should be just one) board = soup.find('a', {"class": "banner-top"}).text board = cleanString(board.strip()) # Finding the repeated tag that corresponds to the listing of topics itopics = soup.find("div", {"class": "postBoard"}).find_all("div", {"class": "item"}, recursive=False) index = 0 for itopic in itopics: # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them # to don't miss any topic # Adding the topic to the topic list topic_title = itopic.find("a", {"class": "title"}) title_flair = topic_title.find('span', {"class": "flair"}) topics = topic_title.text try: title_flair = title_flair.text.strip() topics = topics.replace(title_flair, '') except: pass topics = topics.replace(u'\xa0', ' ') topics = topics.replace(",", "") topics = topics.replace("\n", "") topic.append(cleanString(topics.strip())) # Counting how many topics we have found so far nm = len(topic) # Adding the url to the list of urls link = topic_title['href'] link = cleanLink(link) href.append(link) # Finding the author of the topic ps = itopic.find('div', {"class": "author"}) post_wrapper = ps.select_one('a[href^="/u/"]').text flair = ps.find("span", {"class": "flair"}) try: flair = flair.text.strip() post_wrapper = post_wrapper.replace(flair, '') except: pass author = post_wrapper.strip() user.append(cleanString(author)) # Finding the number of replies meta = itopic.find("div", {"class": "postMain"}) posts = meta.find("a").text posts = posts.replace("comments", '').strip() post.append(cleanString(posts)) # Finding the number of Views - not shown in Dread view.append("-1") # If no information about when the topic was added, just assign "-1" to the variable spans = itopic.find('div', {"class": "author"}).find('span', recursive=False) dt = spans['title'] month = find_month(dt) split_text = dt.split() day = int(re.search(r'\d+', split_text[0]).group()) year = int(split_text[2]) hm = re.findall(r'\d+', split_text[-1]) hm[0] = int(hm[0]) hm[1] = int(hm[1]) date_time_obj = datetime(year, month, day, hour=hm[0], minute=hm[1]) addDate.append(date_time_obj) index += 1 return organizeTopics("Dread", nm, topic, board, view, post, user, addDate, href) def dread_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] #print(soup.find('table', {"class": "tborder clear"}).find( # 'tbody').find_all('tr', {"class": "inline_row"})) listing = soup.find("div", {"class": "postBoard"}).find_all("div",{"class": "item"}, recursive=False) for a in listing: link = a.find("a", {"class": "title"}) link = link['href'] href.append(link) return href def find_month(s): if 'January' in s: return 1 elif 'February' in s: return 2 elif 'March' in s: return 3 elif 'April' in s: return 4 elif 'May' in s: return 5 elif 'June' in s: return 6 elif 'July' in s: return 7 elif 'August' in s: return 8 elif 'September' in s: return 9 elif 'October' in s: return 10 elif 'November' in s: return 11 elif 'December' in s: return 12