|
|
@ -40,7 +40,6 @@ def cryptBB_description_parser(soup): |
|
|
|
|
|
|
|
|
# Finding the repeated tag that corresponds to the listing of posts |
|
|
# Finding the repeated tag that corresponds to the listing of posts |
|
|
|
|
|
|
|
|
# try: |
|
|
|
|
|
posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( |
|
|
posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( |
|
|
'div', {"class": "post"}) |
|
|
'div', {"class": "post"}) |
|
|
|
|
|
|
|
|
@ -48,6 +47,9 @@ def cryptBB_description_parser(soup): |
|
|
|
|
|
|
|
|
for ipost in posts: |
|
|
for ipost in posts: |
|
|
|
|
|
|
|
|
|
|
|
if ipost.find('div', {"class": "deleted_post_author"}): |
|
|
|
|
|
continue |
|
|
|
|
|
|
|
|
# Finding a first level of the HTML page |
|
|
# Finding a first level of the HTML page |
|
|
|
|
|
|
|
|
post_wrapper = ipost.find('span', {"class": "largetext"}) |
|
|
post_wrapper = ipost.find('span', {"class": "largetext"}) |
|
|
@ -61,56 +63,49 @@ def cryptBB_description_parser(soup): |
|
|
|
|
|
|
|
|
smalltext = ipost.find('div', {"class": "post_author"}) |
|
|
smalltext = ipost.find('div', {"class": "post_author"}) |
|
|
|
|
|
|
|
|
''' |
|
|
|
|
|
# Testing here two possibilities to find this status and combine them |
|
|
|
|
|
if ipost.find('div', {"class": "deleted_post_author"}): |
|
|
|
|
|
status.append(-1) |
|
|
|
|
|
interest.append(-1) |
|
|
|
|
|
reputation.append(-1) |
|
|
|
|
|
addDate.append(-1) |
|
|
|
|
|
post.append("THIS POST HAS BEEN REMOVED!") |
|
|
|
|
|
sign.append(-1) |
|
|
|
|
|
feedback.append(-1) |
|
|
|
|
|
continue |
|
|
|
|
|
''' |
|
|
|
|
|
|
|
|
|
|
|
# CryptBB does have membergroup and postgroup |
|
|
|
|
|
|
|
|
if smalltext is not None: |
|
|
|
|
|
|
|
|
membergroup = smalltext.find('div', {"class": "profile-rank"}) |
|
|
|
|
|
postgroup = smalltext.find('div', {"class": "postgroup"}) |
|
|
|
|
|
if membergroup != None: |
|
|
|
|
|
membergroup = membergroup.text.strip() |
|
|
|
|
|
if postgroup != None: |
|
|
|
|
|
postgroup = postgroup.text.strip() |
|
|
|
|
|
membergroup = membergroup + " - " + postgroup |
|
|
|
|
|
else: |
|
|
|
|
|
if postgroup != None: |
|
|
|
|
|
membergroup = postgroup.text.strip() |
|
|
|
|
|
|
|
|
# CryptBB does have membergroup and postgroup |
|
|
|
|
|
membergroup = smalltext.find('div', {"class": "profile-rank"}) |
|
|
|
|
|
postgroup = smalltext.find('div', {"class": "postgroup"}) |
|
|
|
|
|
if membergroup != None: |
|
|
|
|
|
membergroup = membergroup.text.strip() |
|
|
|
|
|
if postgroup != None: |
|
|
|
|
|
postgroup = postgroup.text.strip() |
|
|
|
|
|
membergroup = membergroup + " - " + postgroup |
|
|
else: |
|
|
else: |
|
|
membergroup = "-1" |
|
|
|
|
|
status.append(cleanString(membergroup)) |
|
|
|
|
|
|
|
|
|
|
|
# Finding the interest of the author |
|
|
|
|
|
# CryptBB does not have blurb |
|
|
|
|
|
blurb = smalltext.find('li', {"class": "blurb"}) |
|
|
|
|
|
if blurb != None: |
|
|
|
|
|
blurb = blurb.text.strip() |
|
|
|
|
|
else: |
|
|
|
|
|
blurb = "-1" |
|
|
|
|
|
interest.append(cleanString(blurb)) |
|
|
|
|
|
|
|
|
|
|
|
# Finding the reputation of the user |
|
|
|
|
|
# CryptBB does have reputation |
|
|
|
|
|
author_stats = smalltext.find('div', {"class": "author_statistics"}) |
|
|
|
|
|
karma = author_stats.find('strong') |
|
|
|
|
|
if karma != None: |
|
|
|
|
|
karma = karma.text |
|
|
|
|
|
karma = karma.replace("Community Rating: ", "") |
|
|
|
|
|
karma = karma.replace("Karma: ", "") |
|
|
|
|
|
karma = karma.strip() |
|
|
|
|
|
|
|
|
if postgroup != None: |
|
|
|
|
|
membergroup = postgroup.text.strip() |
|
|
|
|
|
else: |
|
|
|
|
|
membergroup = "-1" |
|
|
|
|
|
status.append(cleanString(membergroup)) |
|
|
|
|
|
|
|
|
|
|
|
# Finding the interest of the author |
|
|
|
|
|
# CryptBB does not have blurb |
|
|
|
|
|
blurb = smalltext.find('li', {"class": "blurb"}) |
|
|
|
|
|
if blurb != None: |
|
|
|
|
|
blurb = blurb.text.strip() |
|
|
|
|
|
else: |
|
|
|
|
|
blurb = "-1" |
|
|
|
|
|
interest.append(cleanString(blurb)) |
|
|
|
|
|
|
|
|
|
|
|
# Finding the reputation of the user |
|
|
|
|
|
# CryptBB does have reputation |
|
|
|
|
|
author_stats = smalltext.find('div', {"class": "author_statistics"}) |
|
|
|
|
|
karma = author_stats.find('strong') |
|
|
|
|
|
if karma != None: |
|
|
|
|
|
karma = karma.text |
|
|
|
|
|
karma = karma.replace("Community Rating: ", "") |
|
|
|
|
|
karma = karma.replace("Karma: ", "") |
|
|
|
|
|
karma = karma.strip() |
|
|
|
|
|
else: |
|
|
|
|
|
karma = "-1" |
|
|
|
|
|
reputation.append(cleanString(karma)) |
|
|
|
|
|
|
|
|
else: |
|
|
else: |
|
|
karma = "-1" |
|
|
|
|
|
reputation.append(cleanString(karma)) |
|
|
|
|
|
|
|
|
status.append('-1') |
|
|
|
|
|
interest.append('-1') |
|
|
|
|
|
reputation.append('-1') |
|
|
|
|
|
|
|
|
# Getting here another good tag to find the post date, post content and users' signature |
|
|
# Getting here another good tag to find the post date, post content and users' signature |
|
|
|
|
|
|
|
|
@ -120,25 +115,30 @@ def cryptBB_description_parser(soup): |
|
|
# dt = dt.strip().split() |
|
|
# dt = dt.strip().split() |
|
|
dt = dt.strip() |
|
|
dt = dt.strip() |
|
|
day=date.today() |
|
|
day=date.today() |
|
|
if "Yesterday" in dt: |
|
|
|
|
|
|
|
|
if "Today" in dt: |
|
|
|
|
|
today = day.strftime('%m-%d-%Y') |
|
|
|
|
|
stime = dt.replace('Today,','').strip() |
|
|
|
|
|
date_time_obj = today + ', '+stime |
|
|
|
|
|
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') |
|
|
|
|
|
elif "Yesterday" in dt: |
|
|
yesterday = day - timedelta(days=1) |
|
|
yesterday = day - timedelta(days=1) |
|
|
yesterday = yesterday.strftime('%m-%d-%Y') |
|
|
yesterday = yesterday.strftime('%m-%d-%Y') |
|
|
stime = dt.replace('Yesterday,','').strip() |
|
|
stime = dt.replace('Yesterday,','').strip() |
|
|
date_time_obj = yesterday+ ', '+stime |
|
|
|
|
|
|
|
|
date_time_obj = yesterday + ', '+stime |
|
|
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') |
|
|
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') |
|
|
elif "hour ago" in dt or "hours ago" in dt: |
|
|
|
|
|
day = day.strftime('%m-%d-%Y') |
|
|
|
|
|
|
|
|
elif "ago" in dt: |
|
|
date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] |
|
|
date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] |
|
|
date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') |
|
|
date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') |
|
|
else: |
|
|
else: |
|
|
date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') |
|
|
date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') |
|
|
stime = date_time_obj.strftime('%b %d, %Y') |
|
|
|
|
|
sdate = date_time_obj.strftime('%I:%M %p') |
|
|
|
|
|
addDate.append(date_time_obj) |
|
|
addDate.append(date_time_obj) |
|
|
|
|
|
|
|
|
# Finding the post |
|
|
# Finding the post |
|
|
|
|
|
|
|
|
inner = postarea.find('div', {"class": "post_body scaleimages"}) |
|
|
inner = postarea.find('div', {"class": "post_body scaleimages"}) |
|
|
|
|
|
quote = inner.find('blockquote') |
|
|
|
|
|
if quote is not None: |
|
|
|
|
|
quote.decompose() |
|
|
inner = inner.text.strip() |
|
|
inner = inner.text.strip() |
|
|
post.append(cleanString(inner)) |
|
|
post.append(cleanString(inner)) |
|
|
|
|
|
|
|
|
@ -210,6 +210,10 @@ def cryptBB_listing_parser(soup): |
|
|
|
|
|
|
|
|
itopics = soup.find_all('tr',{"class": "inline_row"}) |
|
|
itopics = soup.find_all('tr',{"class": "inline_row"}) |
|
|
|
|
|
|
|
|
|
|
|
# Counting how many topics |
|
|
|
|
|
|
|
|
|
|
|
nm = len(itopics) |
|
|
|
|
|
|
|
|
for itopic in itopics: |
|
|
for itopic in itopics: |
|
|
|
|
|
|
|
|
# For each topic found, the structure to get the rest of the information can be of two types. Testing all of them |
|
|
# For each topic found, the structure to get the rest of the information can be of two types. Testing all of them |
|
|
@ -225,10 +229,6 @@ def cryptBB_listing_parser(soup): |
|
|
|
|
|
|
|
|
image_user.append(-1) |
|
|
image_user.append(-1) |
|
|
|
|
|
|
|
|
# Counting how many topics we have found so far |
|
|
|
|
|
|
|
|
|
|
|
nm = len(topic) |
|
|
|
|
|
|
|
|
|
|
|
# Adding the url to the list of urls |
|
|
# Adding the url to the list of urls |
|
|
try: |
|
|
try: |
|
|
link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') |
|
|
link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') |
|
|
@ -237,19 +237,24 @@ def cryptBB_listing_parser(soup): |
|
|
href.append(link) |
|
|
href.append(link) |
|
|
|
|
|
|
|
|
# Finding the author of the topic |
|
|
# Finding the author of the topic |
|
|
ps = itopic.find('div', {"class":"author smalltext"}).find('a').text |
|
|
|
|
|
|
|
|
ps = itopic.find('div', {"class":"author smalltext"}).text |
|
|
user = ps.strip() |
|
|
user = ps.strip() |
|
|
author.append(cleanString(user)) |
|
|
author.append(cleanString(user)) |
|
|
|
|
|
|
|
|
# Finding the number of replies |
|
|
# Finding the number of replies |
|
|
columns = itopic.findChildren('td',recursive=False) |
|
|
columns = itopic.findChildren('td',recursive=False) |
|
|
replies = columns[3].text |
|
|
replies = columns[3].text |
|
|
|
|
|
|
|
|
posts.append(cleanString(replies)) |
|
|
|
|
|
|
|
|
if replies == '-': |
|
|
|
|
|
posts.append('-1') |
|
|
|
|
|
else: |
|
|
|
|
|
posts.append(cleanString(replies)) |
|
|
|
|
|
|
|
|
# Finding the number of Views |
|
|
# Finding the number of Views |
|
|
tview = columns[4].text |
|
|
tview = columns[4].text |
|
|
views.append(cleanString(tview)) |
|
|
|
|
|
|
|
|
if tview == '-': |
|
|
|
|
|
views.append('-1') |
|
|
|
|
|
else: |
|
|
|
|
|
views.append(cleanString(tview)) |
|
|
|
|
|
|
|
|
# If no information about when the topic was added, just assign "-1" to the variable |
|
|
# If no information about when the topic was added, just assign "-1" to the variable |
|
|
|
|
|
|
|
|
|
westernmeadow
1 year ago