diff --git a/Forums/AbyssForum/parser.py b/Forums/AbyssForum/parser.py index 5737610..635c494 100644 --- a/Forums/AbyssForum/parser.py +++ b/Forums/AbyssForum/parser.py @@ -11,231 +11,141 @@ from bs4 import BeautifulSoup # This is the method to parse the Description Pages (one page to each topic in the Listing Pages) -def cryptBB_description_parser(soup): +def abyssForums_description_parser(soup): # Fields to be parsed - topic = "-1" # topic name - user = [] # all users of each post - addDate = [] # all dated of each post - feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) - status = [] # all user's authority in each post such as (adm, member, dangerous) - reputation = [] # all users's karma in each post (usually found as a number) - sign = [] # all user's signature in each post (usually a standard message after the content of the post) - post = [] # all messages of each post - interest = [] # all user's interest in each post + topic = "-1" # 0 topic name + user = [] # 1 all users of each post + addDate = [] # 2 all dated of each post + feedback = [] # 3 all feedbacks of each vendor (this was found in just one Forum and with a number format) + status = [] # 4 all user's authority in each post such as (adm, member, dangerous) + reputation = [] # 5 all users's karma in each post (usually found as a number) + sign = [] # 6 all user's signature in each post (usually a standard message after the content of the post) + post = [] # 7 all messages of each post + interest = [] # 8 all user's interest in each post + image_user = [] # 9 all user avatars of each post + image_post = [] # 10 all first images of each post # Finding the topic (should be just one coming from the Listing Page) - li = soup.find("td", {"class": "thead"}).find('strong') - topic = li.text - topic = re.sub("\[\w*\]", '', topic) - - topic = topic.replace(",","") + li = soup.find("div", {"class": "page-body"}).find("h2", {"class": "topic-title"}) + topic = li.text.replace(",","") topic = topic.replace("\n","") topic = cleanString(topic.strip()) - print(topic) - # Finding the repeated tag that corresponds to the listing of posts - - # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \ - # soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"}) - - try: - posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( - 'div', {"class": "post"}) - # print(len(posts)) - - # For each message (post), get all the fields we are interested to: - - for ipost in posts: - - # Finding a first level of the HTML page - - # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"}) - post_wrapper = ipost.find('span', {"class": "largetext"}) - # Finding the author (user) of the post - - # author = post_wrapper.find('h4') - author = post_wrapper.text.strip() - # print("author " + author) - user.append(cleanString(author)) # Remember to clean the problematic characters - - # Finding the status of the author - - smalltext = ipost.find('div', {"class": "post_author"}) - - # Testing here two possibilities to find this status and combine them - if ipost.find('div', {"class": "deleted_post_author"}): - status.append(-1) - interest.append(-1) - reputation.append(-1) - addDate.append(-1) - post.append("THIS POST HAS BEEN REMOVED!") - sign.append(-1) - feedback.append(-1) - continue - - # CryptBB does have membergroup and postgroup - - membergroup = smalltext.find('div', {"class": "profile-rank"}) - postgroup = smalltext.find('div', {"class": "postgroup"}) - if membergroup != None: - membergroup = membergroup.text.strip() - if postgroup != None: - postgroup = postgroup.text.strip() - membergroup = membergroup + " - " + postgroup - else: - if postgroup != None: - membergroup = postgroup.text.strip() - else: - membergroup = "-1" - - status.append(cleanString(membergroup)) - # print("status " + cleanString(membergroup)) - # Finding the interest of the author - # CryptBB does not have blurb - blurb = smalltext.find('li', {"class": "blurb"}) - if blurb != None: - blurb = blurb.text.strip() - else: - blurb = "-1" - interest.append(cleanString(blurb)) - - # Finding the reputation of the user - # CryptBB does have reputation - author_stats = smalltext.find('div', {"class": "author_statistics"}) - karma = author_stats.find('strong') - if karma != None: - karma = karma.text - karma = karma.replace("Community Rating: ", "") - karma = karma.replace("Karma: ", "") - karma = karma.strip() - else: - karma = "-1" - reputation.append(cleanString(karma)) - # print("karma " + cleanString(karma)) - # Getting here another good tag to find the post date, post content and users' signature - - postarea = ipost.find('div', {"class": "post_content"}) - - dt = postarea.find('span', {"class": "post_date"}).text - # dt = dt.strip().split() - dt = dt.strip() - day=date.today() - if "Yesterday" in dt: - yesterday = day - timedelta(days=1) - yesterday = yesterday.strftime('%m-%d-%Y') - stime = dt.replace('Yesterday,','').strip() - date_time_obj = yesterday+ ', '+stime - date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') - elif "hours ago" in dt: - day = day.strftime('%m-%d-%Y') - date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] - date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') - else: - date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') - stime = date_time_obj.strftime('%b %d, %Y') - sdate = date_time_obj.strftime('%I:%M %p') - - - addDate.append(date_time_obj) - # print("date " + str(date_time_obj)) - # Finding the date of the post - # date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p') - # smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\ - # .find('div', {"class": "smalltext"}) - # sdatetime = smalltext.text - # sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters - # sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters - # sdatetime = sdatetime.split("on: ") # Removing unnecessary characters - # sdatetime = sdatetime[1].strip() - # stime = sdatetime[:-12:-1] # Finding the time of the post - # stime = stime[::-1] - # sdate = sdatetime.replace(stime,"") # Finding the date of the post - # sdate = sdate.replace(",","") - # sdate = sdate.strip() - - # Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need - # a date format here as "mm/dd/yyyy" - - # addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime) - - # Finding the post - - inner = postarea.find('div', {"class": "post_body scaleimages"}) - inner = inner.text.strip() - # print(inner) - post.append(cleanString(inner)) - - # Finding the users's signature - - # signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"}) - signature = ipost.find('div', {"class": "signature scaleimages"}) - if signature != None: - signature = signature.text.strip() - # print(signature) - else: - signature = "-1" - sign.append(cleanString(signature)) - - # As no information about users's feedback was found, just assign "-1" to the variable - - feedback.append("-1") - except: - if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": - user.append("-1") - status.append(-1) - interest.append(-1) - reputation.append(-1) - addDate.append(-1) - post.append("NO ACCESS TO THIS PAGE!") - sign.append(-1) - feedback.append(-1) + regex = re.compile('post has-profile.*') + posts = soup.find_all('div', {"class": regex}) + # print(len(posts)) + + # For each message (post), get all the fields we are interested to: + + for ipost in posts: + + # Finding the author (user) of the post + author = ipost.find('a', {"class": "username"}).text + user.append(cleanString(author)) # Remember to clean the problematic characters + + status.append("-1") + reputation.append("-1") + interest.append("-1") + sign.append("-1") + feedback.append("-1") + image_post.append("-1") + + img = ipost.find('dl', {"class": "postprofile"}).find('img') + if img is not None: + img = img.get('src').split('base64,')[-1] + else: + img = "-1" + image_user.append(img) + + image_user.append("-1") + + date_time_obj = ipost.find('time').attrs + date = date_time_obj['datetime'][0:10] + time = date_time_obj['datetime'][11:19] + date_time_obj = datetime.strptime(date + " " + time, '%Y-%m-%d %H:%M:%S') + addDate.append(date_time_obj) + + # Finding the post + + inner = ipost.find('div', {"class": "content"}) + inner = inner.text.strip() + post.append(cleanString(inner)) # Populate the final variable (this should be a list with all fields scraped) - row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) + row = (topic, user, status, reputation, interest, sign, post, feedback, addDate, image_user, image_post) # Sending the results return row # This is the method to parse the Listing Pages (one page with many posts) -def AbyssForums_listing_parser(soup: BeautifulSoup): - board = "-1" # board name (the previous level of the topic in the Forum categorization tree. - # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - - nm = 0 # this variable should receive the number of topics - topic = [] # all topics - user = [] # all users of each topic - post = [] # number of posts of each topic - view = [] # number of views of each topic - addDate = [] # when the topic was created (difficult to find) - href = [] # this variable should receive all cleaned urls (we will use this to do the marge between +def abyssForums_listing_parser(soup: BeautifulSoup): + + + nm = 0 # this variable should receive the number of topics + forum = "AbyssForum" # 0 *forum name + board = "-1" # 1 board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + author = [] # 2 all authors of each topic + topic = [] # 3 all topics + views = [] # 4 number of views of each topic + posts = [] # 5 number of posts of each topic + href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between + addDate = [] # when the topic was created (difficult to find) + image_author = [] # 8 all author avatars used in each topic + # Listing and Description pages) #finding the board - board = soup.find("title").text + board = soup.find("h2", {"class": "forum-title"}).text board = cleanString(board.strip()) type_of_posts = soup.find_all("li", {"class": re.compile("row bg\d")} ) for literature in type_of_posts: title_of_post = literature.find("a", {"class": "topictitle"}).text + title_of_post = cleanString(title_of_post) topic.append(title_of_post) - author = literature.find("div", {"class": "topic-poster responsive-hide left-box"}).find("a", {"class": "username"}).text - user.append(author) - num_post = literature.find("dd", {"class": "posts"}).text[1:-3] - post.append(num_post) - num_view = literature.find("dd", {"class": "views"}).text[1:-3] - view.append(num_view) - if int(num_post) != 0: - reply = literature.find("dd", {"class": "lastpost"}).find("a", {"class": "username"}).text - user.append(reply) - date_added = literature.find("time").text + user = literature.find("div", {"class": "topic-poster responsive-hide left-box"}).find("a", {"class": "username"}).text + author.append(user) + num_post = literature.find("dd", {"class": "posts"}).text.replace("Replies","").strip() + posts.append(num_post) + num_view = literature.find("dd", {"class": "views"}).text.replace("Views","").strip() + views.append(num_view) + #if int(num_post) != 0: join the last user who posted with the author? + # reply = literature.find("dd", {"class": "lastpost"}).find("a", {"class": "username"}).text + # user.append(reply) + + date_time_obj = literature.find('time').attrs + date = date_time_obj['datetime'][0:10] + time = date_time_obj['datetime'][11:19] + date_added = datetime.strptime(date + " " + time, '%Y-%m-%d %H:%M:%S') + addDate.append(date_added) + + listing_href = literature.find("a", {"class": "topictitle"}).get("href") + href.append(listing_href) + + image_author.append("-1") + nm = len(topic) + return organizeTopics( + forum=forum, + nm=nm, + board=board, + author=author, + topic=topic, + views=views, + posts=posts, + href=href, + addDate=addDate, + image_author=image_author + ) + diff --git a/Forums/Altenens/parser.py b/Forums/Altenens/parser.py index 70abf79..8ea1fe0 100644 --- a/Forums/Altenens/parser.py +++ b/Forums/Altenens/parser.py @@ -100,7 +100,7 @@ def altenens_listing_parser(soup): href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between # Listing and Description pages) addDate = [] # 7 when the topic was created (difficult to find) - image_user = [] # 8 all user avatars used in each topic + image_author = [] # 8 all author avatars used in each topic board = soup.find('h1', {"class": "p-title-value"}).text board = cleanString(board.strip()) @@ -122,7 +122,7 @@ def altenens_listing_parser(soup): author_icon = author_icon.split('base64,')[-1] else: author_icon = "-1" - image_user.append(author_icon) + image_author.append(author_icon) link = itopic.find('div', {"class": "structItem-title"}).find('a').get('href') href.append(link) @@ -144,7 +144,7 @@ def altenens_listing_parser(soup): nviews = nviews.replace('K', '000') views.append(cleanString(nviews)) - return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_user) + return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_author) def altenens_links_parser(soup): diff --git a/Forums/Cardingleaks/parser.py b/Forums/Cardingleaks/parser.py index 022fbe1..56f7ba7 100644 --- a/Forums/Cardingleaks/parser.py +++ b/Forums/Cardingleaks/parser.py @@ -102,7 +102,7 @@ def cardingleaks_listing_parser(soup: Tag): href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between # Listing and Description pages) addDate = [] # 7 when the topic was created (difficult to find) - image_user = [] # 8 all user avatars used in each topic + image_author = [] # 8 all author avatars used in each topic # Finding the board (should be just one) @@ -123,7 +123,7 @@ def cardingleaks_listing_parser(soup: Tag): author_icon = thread.find("a", {"class": "avatar avatar--s"}).find("img") author_icon = author_icon.get('src') author_icon = author_icon.split('base64,')[-1] - image_user.append(author_icon) + image_author.append(author_icon) thread_view = thread.find("dl", {"class": "pairs pairs--justified structItem-minor"}).find("dd").text # Context text view count (i.e., 8.8K) to numerical (i.e., 8800) @@ -141,7 +141,7 @@ def cardingleaks_listing_parser(soup: Tag): datetime_obj = datetime.strptime(thread_date, "%Y-%m-%dT%H:%M:%S%z") addDate.append(datetime_obj) - return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_user) + return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_author) def cardingleaks_links_parser(soup): diff --git a/Forums/CryptBB/parser.py b/Forums/CryptBB/parser.py index d725a98..f1ca877 100644 --- a/Forums/CryptBB/parser.py +++ b/Forums/CryptBB/parser.py @@ -198,7 +198,7 @@ def cryptBB_listing_parser(soup): href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between # Listing and Description pages) addDate = [] # 7 when the topic was created (difficult to find) - image_user = [] # 8 all user avatars used in each topic + image_author = [] # 8 all author avatars used in each topic # Finding the board (should be just one) @@ -223,7 +223,7 @@ def cryptBB_listing_parser(soup): topics = re.sub("\[\w*\]", '', topics) topic.append(cleanString(topics)) - image_user.append(-1) + image_author.append(-1) # Counting how many topics we have found so far @@ -255,7 +255,7 @@ def cryptBB_listing_parser(soup): addDate.append("-1") - return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_user) + return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate, image_author) def cryptBB_links_parser(soup): diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index eebc0da..e4f6c5d 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -2,6 +2,7 @@ __author__ = 'DarkWeb' import psycopg2 import traceback +from Forums.Utilities.utilities import * def connectDataBase(): @@ -27,7 +28,7 @@ def verifyForum(cur, nameForum): try: - cur.execute("lock table forums IN ACCESS EXCLUSIVE MODE;") + cur.execute("lock table forums IN ACCESS EXCLUSIVE MODE") cur.execute("select forum_id from forums where name_forum = %(nameForum)s limit 1", {'nameForum': nameForum}) @@ -48,7 +49,7 @@ def verifyTopic(cur, forumId, authorId, titleTopic): try: - cur.execute("lock table topics IN ACCESS EXCLUSIVE MODE;") + cur.execute("lock table topics IN ACCESS EXCLUSIVE MODE") cur.execute("select topic_id from topics where forum_id = %(forumId)s and author_id = %(authorId)s and title_topic = %(titleTopic)s limit 1", {'forumId': forumId, 'authorId': authorId, 'titleTopic': titleTopic}) @@ -70,7 +71,7 @@ def verifyPost(cur, topicId, userId, dateAdded): try: - cur.execute("lock table posts IN ACCESS EXCLUSIVE MODE;") + cur.execute("lock table posts IN ACCESS EXCLUSIVE MODE") cur.execute("select post_id from posts where topic_id = %(topicId)s and " "user_id = %(userId)s and dateadded_post = %(dateAdded)s limit 1", {'topicId': topicId, @@ -94,7 +95,7 @@ def verifyUser(cur, nameUser, forumId): try: - cur.execute("lock table users IN ACCESS EXCLUSIVE MODE;") + cur.execute("lock table users IN ACCESS EXCLUSIVE MODE") cur.execute("select user_id from users where name_user = %(nameUser)s and forum_id = %(forumId)s limit 1", {'nameUser': nameUser, 'forumId': forumId}) @@ -377,7 +378,7 @@ def create_user(cur, row, forumId, index): recset = cur.fetchall() - # decode_decrypt_image_in_base64(recset[0][7]) + #decode_decrypt_image_in_base64(recset[0]['image_user']) if (str(recset[0]['status_user']) != str(row[11][index] if row[11][index] != '-1' else None) or str(recset[0]['reputation_user']) != str(row[12][index] if row[12][index] != '-1' else None) or @@ -459,6 +460,8 @@ def create_posts(cur, row, forumId, topicId): str(recset[0]['feedback_post']) != str(row[16][i] if row[16][i] != '-1' else None) or str(recset[0]['image_post']) != str(row[18][i] if row[18][i] != '-1' else None)): # there was a change in the post information + #decode_decrypt_image_in_base64(recset[0]['image_post']) + postVersionId = int(getLastPostVersion(cur, postId) + 1) sql = "Insert into posts_history (post_id, version_post, topic_id, user_id, content_post, feedback_post, " \ diff --git a/Forums/HiddenAnswers/parser.py b/Forums/HiddenAnswers/parser.py index 995a7f0..8b72c12 100644 --- a/Forums/HiddenAnswers/parser.py +++ b/Forums/HiddenAnswers/parser.py @@ -12,7 +12,7 @@ from bs4 import BeautifulSoup, ResultSet, Tag # This is the method to parse the Description Pages (one page to each topic in the Listing Pages) -def HiddenAnswers_description_parser(soup: BeautifulSoup): +def hiddenAnswers_description_parser(soup: BeautifulSoup): # Fields to be parsed @@ -123,7 +123,7 @@ def HiddenAnswers_description_parser(soup: BeautifulSoup): return row -def HiddenAnswers_listing_parser(soup: BeautifulSoup): +def hiddenAnswers_listing_parser(soup: BeautifulSoup): nm: int = 0 # this variable should receive the number of topics forum: str = "HiddenAnswers" # 0 *forum name @@ -136,7 +136,7 @@ def HiddenAnswers_listing_parser(soup: BeautifulSoup): href: List[str] = [] # 6 this variable should receive all cleaned urls (we will use this to do the merge between # Listing and Description pages) addDate: List[str] = [] # 7 when the topic was created (difficult to find) - image_user = [] # 8 all user avatars used in each topic + image_author = [] # 8 all author avatars used in each topic # Finding the board literature = soup.find("div", {"class": "qa-main-heading"}).find("h1") @@ -148,7 +148,7 @@ def HiddenAnswers_listing_parser(soup: BeautifulSoup): topic_of_query = queries.find("div", {"class": "qa-q-item-title"}).find("a").text topic.append(cleanString(topic_of_query.strip())) - image_user.append("-1") + image_author.append("-1") author = queries.find("span", {"class": "qa-q-item-who-data"}).find("a").text user.append(cleanString(author.strip())) @@ -175,7 +175,7 @@ def HiddenAnswers_listing_parser(soup: BeautifulSoup): nm = len(topic) - return organizeTopics(forum, nm, board, user, topic, view, post, href, addDate, image_user) + return organizeTopics(forum, nm, board, user, topic, view, post, href, addDate, image_author) #need to change this method def hiddenanswers_links_parser(soup): diff --git a/Forums/Libre/parser.py b/Forums/Libre/parser.py index 7783661..80aa790 100644 --- a/Forums/Libre/parser.py +++ b/Forums/Libre/parser.py @@ -14,15 +14,17 @@ from bs4 import BeautifulSoup, ResultSet, Tag def libre_description_parser(soup: Tag): # Fields to be parsed - topic = "-1" # 0 *topic name - user = [] # 1 *all users of each post - status = [] # 2 all user's authority in each post such as (adm, member, dangerous) - reputation = [] # 3 all user's karma in each post (usually found as a number) - interest = [] # 4 all user's interest in each post - sign = [] # 5 all user's signature in each post (usually a standard message after the content of the post) - post = [] # 6 all messages of each post - feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format) - addDate = [] # 8 all dates of each post + topic = "-1" # 0 *topic name + user = [] # 1 *all users of each post + status = [] # 2 all user's authority in each post such as (adm, member, dangerous) + reputation = [] # 3 all user's karma in each post (usually found as a number) + interest = [] # 4 all user's interest in each post + sign = [] # 5 all user's signature in each post (usually a standard message after the content of the post) + post = [] # 6 all messages of each post + feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format) + addDate = [] # 8 all dates of each post + image_user = [] # 9 all user avatars of each post + image_post = [] # 10 all first images of each post # Finding the topic (should be just one coming from the Listing Page) @@ -52,6 +54,15 @@ def libre_description_parser(soup: Tag): sign.append("-1") feedback.append("-1") + image_post.append("-1") + + img = original_post.find('img') + if img is not None: + img = img.get('src').split('base64,')[-1] + else: + img = "-1" + image_user.append(img) + # Finding the repeated tag that corresponds to the listing of posts # try: @@ -100,6 +111,14 @@ def libre_description_parser(soup: Tag): feedback.append("-1") + # As no information about post's image was found, just assign "-1" to the variable + + image_post.append("-1") + + # As no information about user's image was found, just assign "-1" to the variable + + image_user.append("-1") + # Populate the final variable (this should be a list with all fields scraped) # print(topic) # print(user) @@ -118,7 +137,7 @@ def libre_description_parser(soup: Tag): # print(len(feedback)) # print(len(addDate)) - row = (topic, user, status, reputation, interest, sign, post, feedback, addDate) + row = (topic, user, status, reputation, interest, sign, post, feedback, addDate, image_user, image_post) # Sending the results @@ -127,17 +146,18 @@ def libre_description_parser(soup: Tag): # This is the method to parse the Listing Pages (one page with many posts) def libre_listing_parser(soup): - nm = 0 # *this variable should receive the number of topics - forum = "Libre" # 0 *forum name - board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree. + nm = 0 # *this variable should receive the number of topics + forum = "Libre" # 0 *forum name + board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree. # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - author = [] # 2 *all authors of each topic - topic = [] # 3 *all topics - views = [] # 4 number of views of each topic - posts = [] # 5 number of posts of each topic - href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between + author = [] # 2 *all authors of each topic + topic = [] # 3 *all topics + views = [] # 4 number of views of each topic + posts = [] # 5 number of posts of each topic + href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between # Listing and Description pages) - addDate = [] # 7 when the topic was created (difficult to find) + addDate = [] # 7 when the topic was created (difficult to find) + image_author = [] # 8 all author avatars used in each topic # Finding the board (should be just one) @@ -159,6 +179,8 @@ def libre_listing_parser(soup): cleaned_topic_string = cleanString(topic_string.strip()) topic.append(cleaned_topic_string) + image_author.append("-1") + # Adding the url to the list of urls link_to_clean = itopic.find("a", {"class": "link text-xl text-zinc-300"}).get("href") @@ -209,7 +231,8 @@ def libre_listing_parser(soup): views=views, posts=posts, href=href, - addDate=addDate + addDate=addDate, + image_author=image_author ) diff --git a/Forums/OnniForums/parser.py b/Forums/OnniForums/parser.py index e0c780a..98cfb68 100644 --- a/Forums/OnniForums/parser.py +++ b/Forums/OnniForums/parser.py @@ -15,15 +15,17 @@ from bs4 import BeautifulSoup def onniForums_description_parser(soup: BeautifulSoup) -> tuple: - topicName: str = "-1" # 0 *topic name - users : List[str] = [] # 1 *all users of each post - statuses : List[str] = [] # 2 all user's authority in each post such as (adm, member, dangerous) - reputations : List[str] = [] # 3 all user's karma in each post (usually found as a number) - interests : List[str] = [] # 4 all user's interest in each post - signs : List[str] = [] # 5 all user's signature in each post (usually a standard message after the content of the post) - posts : List[str] = [] # 6 all messages of each post - feedbacks : List[str] = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format) + topicName: str = "-1" # 0 *topic name + users : List[str] = [] # 1 *all users of each post + statuses : List[str] = [] # 2 all user's authority in each post such as (adm, member, dangerous) + reputations : List[str] = [] # 3 all user's karma in each post (usually found as a number) + interests : List[str] = [] # 4 all user's interest in each post + signs : List[str] = [] # 5 all user's signature in each post (usually a standard message after the content of the post) + posts : List[str] = [] # 6 all messages of each post + feedbacks : List[str] = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format) addDates : List[datetime] = [] # 8 all dates of each post + image_user : List[str] = [] # 9 all user avatars of each post + image_post : List[str] = [] # 10 all first images of each post # Getting the topicName topicName = soup.find("table", {"class": "tborder tfixed clear"}) \ @@ -99,14 +101,21 @@ def onniForums_description_parser(soup: BeautifulSoup) -> tuple: date_object = datetime.strptime(date_posted_cleaned, "%m-%d-%Y") addDates.append(date_object) - - + + image_post.append("-1") + + img = topic.find('div', {"class": "author_avatar"}).find('img') + if img is not None: + img = img.get('src').split('base64,')[-1] + else: + img = "-1" + image_user.append(img) # TESTING PURPOSES - DO NOT REMOVE # Populate the final variable (this should be a list with all fields scraped) - row = (topicName, users, statuses, reputations, interests, signs, posts, feedbacks, addDates) + row = (topicName, users, statuses, reputations, interests, signs, posts, feedbacks, addDates, image_user, image_post) # Sending the results @@ -115,18 +124,18 @@ def onniForums_description_parser(soup: BeautifulSoup) -> tuple: def onniForums_listing_parser(soup: BeautifulSoup): - - boardName = "-1" # board name (the previous level of the topic in the Forum categorization tree. - # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - forum = "OnniForums" - nm = 0 # this variable should receive the number of topics - topic : List[str] = [] # all topics - user : List[str] = [] # all users of each topic - post : List[int] = [] # number of posts of each topic - view : List[int] = [] # number of views of each topic - addDate : List[str] = [] # when the topic was created (difficult to find) - href : List[str] = [] # this variable should receive all cleaned urls (we will use this to do the merge between - # Listing and Description pages) + + nm = 0 # this variable should receive the number of topics + forum = "OnniForums" # 0 *forum name + boardName = "-1" # 1 board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + user: List[str] = [] # 2 all users of each topic + topic : List[str] = [] # 3 all topics + view: List[int] = [] # 4 number of views of each topic + post : List[int] = [] # 5 number of posts of each topic + href: List[str] = [] # 6 this variable should receive all cleaned urls (we will use this to do the merge between Listing and Description pages) + addDate : List[str] = [] # 7 when the topic was created (difficult to find) + image_author : List[str] = [] # 8 all author avatars used in each topic # Finding the board (should be just one) board_metadata: BeautifulSoup = soup.find("table",{"class" : "tborder clear"}) @@ -149,8 +158,16 @@ def onniForums_listing_parser(soup: BeautifulSoup): post_subject_cleaned = cleanString(post_subject.strip()) topic.append(post_subject_cleaned) - - + + author_icon = thread.find('div', {"class": "lavatar-old lavatar-old-f"}) + if author_icon != None: + author_icon = author_icon.find('img') + author_icon = author_icon.get('src') + author_icon = author_icon.split('base64,')[-1] + else: + author_icon = "-1" + image_author.append(author_icon) + reply_count = thread.find_all("td", {"align": "center"})[2].text post.append(cleanNumbers(reply_count)) @@ -177,7 +194,8 @@ def onniForums_listing_parser(soup: BeautifulSoup): views=view, posts=post, href=href, - addDate=addDate + addDate=addDate, + image_author=image_author ) diff --git a/Forums/Procrax/crawler_selenium.py b/Forums/Procrax/crawler_selenium.py index 7115d6c..be48532 100644 --- a/Forums/Procrax/crawler_selenium.py +++ b/Forums/Procrax/crawler_selenium.py @@ -42,11 +42,7 @@ def startCrawling(): print(driver.current_url, e) closeDriver(driver) - new_parse( - forum=FORUM_NAME, - url=BASE_URL, - createLog=True - ) + new_parse(forum=FORUM_NAME, url=BASE_URL, createLog=True) # Login using premade account credentials and do login captcha manually diff --git a/Forums/Procrax/parser.py b/Forums/Procrax/parser.py index 117fdca..cb16271 100644 --- a/Forums/Procrax/parser.py +++ b/Forums/Procrax/parser.py @@ -16,15 +16,17 @@ def procrax_description_parser(soup: Tag): # Fields to be parsed - topic = "-1" # topic name - user = [] # all users of each post - addDate = [] # all dated of each post - feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) - status = [] # all user's authority in each post such as (adm, member, dangerous) - reputation = [] # all user's karma in each post (usually found as a number) - sign = [] # all user's signature in each post (usually a standard message after the content of the post) - post = [] # all messages of each post - interest = [] # all user's interest in each post + topic = "-1" # 0 topic name + user = [] # 1 all users of each post + addDate = [] # 2 all dated of each post + feedback = [] # 3 all feedbacks of each vendor (this was found in just one Forum and with a number format) + status = [] # 4 all user's authority in each post such as (adm, member, dangerous) + reputation = [] # 5 all user's karma in each post (usually found as a number) + sign = [] # 6 all user's signature in each post (usually a standard message after the content of the post) + post = [] # 7 all messages of each post + interest = [] # 8 all user's interest in each post + image_user = [] # 9 all user avatars of each post + image_post = [] # 10 all first images of each post # Finding the topic (should be just one coming from the Listing Page) @@ -40,8 +42,7 @@ def procrax_description_parser(soup: Tag): date_posted = ipost.find("ul", {"class": "message-attribution-main listInline"}).find("time").get("datetime") datetime_obj = datetime.strptime(date_posted, "%Y-%m-%dT%H:%M:%S%z") addDate.append(datetime_obj) - - + feedback.append("-1") user_status = ipost.find("h5", {"class": "userTitle message-userTitle"}).text @@ -57,11 +58,31 @@ def procrax_description_parser(soup: Tag): interest.append("-1") - + bbWrapper = ipost.find('div', {"class": "bbWrapper"}) + if bbWrapper is not None: + img = bbWrapper.find('img') + if img is not None: + img = img.get('src').split('base64,')[-1] + else: + img = "-1" + else: + img = "-1" + image_post.append(img) + + avatar = ipost.find("a", {"class": "avatar avatar--m"}) + if avatar is not None: + img = avatar.find('img') + if img is not None: + img = img.get('src').split('base64,')[-1] + else: + img = "-1" + else: + img = "-1" + image_user.append(img) # Populate the final variable (this should be a list with all fields scraped) - row = (topic, user, status, reputation, interest, sign, post, feedback, addDate) + row = (topic, user, status, reputation, interest, sign, post, feedback, addDate, image_user, image_post) # Sending the results @@ -71,17 +92,19 @@ def procrax_description_parser(soup: Tag): def procrax_listing_parser(soup: Tag): - board = "-1" # board name (the previous level of the topic in the Forum categorization tree. - # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + nm = 0 # this variable should receive the number of topics + forum: str = "Procrax" # 0 *forum name + board = "-1" # 1 board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - nm = 0 # this variable should receive the number of topics - topic = [] # all topics - author = [] # all authors of each topic - views = [] # number of views of each topic - posts = [] # number of posts of each topic - addDate = [] # when the topic was created (difficult to find) - href = [] # this variable should receive all cleaned urls (we will use this to do the marge between - # Listing and Description pages) + author = [] # 2 all authors of each topic + topic = [] # 3 all topics + views = [] # 4 number of views of each topic + posts = [] # 5 number of posts of each topic + href = [] # 6this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + addDate = [] # 7 when the topic was created (difficult to find) + image_author = [] # 8 all author avatars used in each topic # Finding the board (should be just one) li = soup.find("h1", {"class": "p-title-value"}) @@ -94,11 +117,24 @@ def procrax_listing_parser(soup: Tag): for thread in threads_list: thread_title = thread.find("div", {"class": "structItem-title"}).text topic.append(cleanString(thread_title.strip())) - + + author_icon = thread.find('a', {"class": "avatar avatar--s"}) + if author_icon != None: + author_icon = author_icon.find('img') + if author_icon != None: + author_icon = author_icon.get('src') + author_icon = author_icon.split('base64,')[-1] + else: + author_icon = "-1" + else: + author_icon = "-1" + image_author.append(author_icon) + thread_author = thread.get("data-author") author.append(cleanString(thread_author)) thread_views = thread.find("dl", {"class": "pairs pairs--justified structItem-minor"}).find('dd').text + thread_views = thread_views.lower().replace("k","000") views.append(cleanString(thread_views.strip())) thread_replies = thread.find("dl", {"class": "pairs pairs--justified"}).find('dd').text @@ -115,7 +151,7 @@ def procrax_listing_parser(soup: Tag): return organizeTopics( - forum="Procrax", + forum=forum, nm=nm, board=board, author=author, @@ -123,7 +159,8 @@ def procrax_listing_parser(soup: Tag): views=views, posts=posts, addDate=addDate, - href=href + href=href, + image_author=image_author ) diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index b8ef27a..46c05cd 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -223,7 +223,7 @@ def create_vendor(cur, row, marketId): recset = cur.fetchall() - # decode_decrypt_image_in_base64(recset[0][5]) + decode_decrypt_image_in_base64(recset[0][5]) if (str(recset[0]['rating_vendor']) != str(row[2] if row[2] != '-1' else None) or # there was a change in the vendor information str(recset[0]['successfultransactions_vendor']) != str(row[3] if row[3] != '-1' else None) or