dark_web_forums/Forums/HiddenAnswers/parser.py

__author__ = 'Helium'

# Here, we are importing the auxiliary functions to clean or convert data
from typing import List
from Forums.Utilities.utilities import *
from datetime import date
from datetime import timedelta
import re

# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup, ResultSet, Tag

# This is the method to parse the Description Pages (one page to each topic in the Listing Pages)

def HiddenAnswers_description_parser(soup: BeautifulSoup):

    # Fields to be parsed

    topic: str = "-1"              # 0 topic name
    user: List[str] = []           # 1 all users of each post
    addDate: List[datetime] = []   # 2 all dated of each post
    feedback: List[str] = []       # 3 all feedbacks of each vendor (this was found in just one Forum and with a number format)
    status: List[str] = []         # 4 all user's authority in each post such as (adm, member, dangerous)
    reputation: List[str] = []     # 5 all user's karma in each post (usually found as a number)
    sign: List[str] = []           # 6 all user's signature in each post (usually a standard message after the content of the post)
    post: List[str] = []           # 7 all messages of each post
    interest: List[str] = []       # 8 all user's interest in each post
    image_user = []                # 9 all user avatars of each post
    image_post = []                # 10 all first images of each post

    # Finding the topic (should be just one coming from the Listing Page)
    li = soup.find("h1").find("span", {"itemprop": "name"})
    topic = li.text

    question: Tag = soup.find("div", {"class": "qa-part-q-view"})
    
    question_user = question.find("span", {"class": "qa-q-view-who-data"}).text
    user.append(cleanString(question_user.strip()))
    
    question_time = question.find("span", {"class": "qa-q-view-when-data"}).find("time").get("datetime")
    datetime_string = question_time.split("+")[0]
    datetime_obj = datetime.strptime(datetime_string, "%Y-%m-%dT%H:%M:%S")
    addDate.append(datetime_obj)
    
    question_user_status = question.find("span", {"class": "qa-q-view-who-title"})
    if question_user_status is not None:
        question_user_status = question_user_status.text
        status.append(cleanString(question_user_status.strip()))
    else:
        status.append('-1')

    question_user_karma = question.find("span", {"class": "qa-q-view-who-points-data"})
    if question_user_karma is not None:
        question_user_karma = question_user_karma.text
        # Convert karma to pure numerical string
        if question_user_karma.find("k") > -1:
            question_user_karma = str(float(question_user_karma.replace("k", "")) * 1000)
        reputation.append(cleanString(question_user_karma.strip()))
    else:
        reputation.append('-1')
    
    question_content = question.find("div", {"class": "qa-q-view-content qa-post-content"}).text
    post.append(cleanString(question_content.strip()))
    
    feedback.append("-1")
    sign.append("-1")
    interest.append("-1")

    img = question.find('div', {"class": "qa-q-view-content qa-post-content"}).find('img')
    if img is not None:
        img = img.get('src').split('base64,')[-1]
    else:
        img = "-1"
    image_post.append(img)

    img = question.find('span', {"class": "qa-q-view-avatar-meta"}).find('img')
    if img is not None:
        img = img.get('src').split('base64,')[-1]
    else:
        img = "-1"
    image_user.append(img)
    
    answer_list: ResultSet[Tag] = soup.find("div", {"class": "qa-a-list"}).find_all("div", {"class": "qa-a-list-item"})
    
    
    for replies in answer_list:
        user_name = replies.find("span", {"class", "qa-a-item-who-data"}).text
        user.append(cleanString(user_name.strip()))
        
        date_added = replies.find("span", {"class": "qa-a-item-when"}).find("time", {"itemprop": "dateCreated"}).get('datetime')
        date_string = date_added.split("+")[0]
        datetime_obj = datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
        addDate.append(datetime_obj)
        
        
        post_data = replies.find("div", {"class": "qa-a-item-content qa-post-content"}).find("div",{"itemprop":"text"}).text
        post.append(cleanString(post_data.strip()))
        
        user_reputations = replies.find("span", {"class", "qa-a-item-who-title"})
        if user_reputations is not None:
            user_reputations = user_reputations.text
            status.append(cleanString(user_reputations.strip()))
        else:
            status.append('-1')
        
        karma = replies.find("span", {"class": "qa-a-item-who-points-data"})
        if karma is not None:
            karma = karma.text
            # Convert karma to pure numerical string
            if karma.find("k") > -1:
                karma = str(float(karma.replace("k", "")) * 1000)
            reputation.append(cleanString(karma.strip()))
        else:
            reputation.append('-1')
        
        feedback.append("-1")
        sign.append("-1")
        interest.append("-1")

        img = replies.find("div", {"class": "qa-a-item-content qa-post-content"}).find("div",{"itemprop":"text"}).find('img')
        if img is not None:
            img = img.get('src').split('base64,')[-1]
        else:
            img = "-1"
        image_post.append(img)

        img = replies.find('span', {"class": "qa-a-item-avatar-meta"}).find('img')
        if img is not None:
            img = img.get('src').split('base64,')[-1]
        else:
            img = "-1"
        image_user.append(img)

    # Populate the final variable (this should be a list with all fields scraped)

    row = (topic, user, status, reputation, interest, sign, post, feedback, addDate, image_user, image_post)

    # Sending the results
    return row


def HiddenAnswers_listing_parser(soup: BeautifulSoup):

    nm: int = 0                    # this variable should receive the number of topics
    forum: str = "HiddenAnswers"   # 0 *forum name
    board = "-1"                   # 1 board name (the previous level of the topic in the Forum categorization tree.
                                   # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
    user: List[str] = []           # 2 all users of each topic
    topic: List[str] = []          # 3 all topics
    view: List[int] = []           # 4 number of views of each topic
    post: List[int] = []           # 5 number of posts of each topic
    href: List[str] = []           # 6 this variable should receive all cleaned urls (we will use this to do the merge between
                                   # Listing and Description pages)
    addDate: List[str] = []        # 7 when the topic was created (difficult to find)
    image_user = []                # 8 all user avatars used in each topic

    # Finding the board
    board = soup.find("div", {"class": "qa-main-heading"}).find("h1").text
    board = board.replace('Recent questions in', '')
    board = cleanString(board.strip())
    
    queries_by_user: ResultSet[Tag] = soup.find("div", {"class": "qa-q-list"}).find_all("div", {"class": "qa-q-list-item"})
    
    for queries in queries_by_user:
        topic_of_query = queries.find("div", {"class": "qa-q-item-title"}).find("a").text
        topic.append(cleanString(topic_of_query.strip()))

        image_user.append("-1") # qa-q-item-where
        
        author = queries.find("span", {"class": "qa-q-item-who-data"}).text
        user.append(cleanString(author.strip()))
        
        num_answers = queries.find("span", {"class": "qa-a-count-data"}).text
        post.append(cleanString(num_answers.strip()))
        
        view.append("-1")
        
        date_posted = queries.find("span", {"class": "qa-q-item-when-data"}).text
        
        if date_posted.find("day") > 0:
            datetime_obj = datetime.now() - timedelta(days=1)
        else:
            try:
                datetime_obj = datetime.strptime(f"{date_posted} {date.today().year}", "%b %d %Y")
            except ValueError:
                datetime_obj = datetime.strptime(f"{date_posted}", "%b %d, %Y")
        addDate.append(datetime_obj)
        #this link will be cleaned
        
        listing_href = queries.find("div", {"class": "qa-q-item-title"}).find("a").get("href")
        href.append(listing_href)
        
    nm = len(topic)

    return organizeTopics(forum, nm, board, user, topic, view, post, href, addDate, image_user)

#need to change this method
def hiddenanswers_links_parser(soup):

    # Returning all links that should be visited by the Crawler

    href = []
    #print(soup.find('table', {"class": "tborder clear"}).find(
     #   'tbody').find_all('tr', {"class": "inline_row"}))
    listing = soup.find_all('div', {"class": "qa-q-item-title"})

    for a in listing:
        link = a.find('a').get('href')

        href.append(link)

    return href