From 2d5116968d0730979b2229c58fb246b406aa0974 Mon Sep 17 00:00:00 2001
From: westernmeadow <wkwan626@gmail.com>
Date: Mon, 17 Jul 2023 15:15:53 -0700
Subject: [PATCH] updated all marketplace crawlers
---
Forums/AbyssForum/crawler_selenium.py | 68 ++++++-------------
.../AnonymousMarketplace/crawler_selenium.py | 35 ++++------
MarketPlaces/Apocalypse/crawler_selenium.py | 33 +++------
MarketPlaces/BlackPyramid/crawler_selenium.py | 33 +++------
MarketPlaces/CityMarket/crawler_selenium.py | 29 +++-----
.../CypherMarketplace/crawler_selenium.py | 29 +++-----
MarketPlaces/DarkFox/crawler_selenium.py | 44 ++++++------
MarketPlaces/DarkMatter/crawler_selenium.py | 30 +++-----
MarketPlaces/DarkTor/crawler_selenium.py | 33 +++------
.../DigitalThriftShop/crawler_selenium.py | 29 +++-----
.../LionMarketplace/crawler_selenium.py | 29 +++-----
.../M00nkeyMarket/crawler_selenium.py | 32 +++------
.../MikesGrandStore/crawler_selenium.py | 35 ++++------
MarketPlaces/ThiefWorld/crawler_selenium.py | 31 +++------
MarketPlaces/Tor2door/crawler_selenium.py | 32 +++------
MarketPlaces/TorBay/crawler_selenium.py | 41 ++++-------
MarketPlaces/TorMarket/crawler_selenium.py | 29 +++-----
17 files changed, 199 insertions(+), 393 deletions(-)
diff --git a/Forums/AbyssForum/crawler_selenium.py b/Forums/AbyssForum/crawler_selenium.py
index 573cd13..32678dc 100644
--- a/Forums/AbyssForum/crawler_selenium.py
+++ b/Forums/AbyssForum/crawler_selenium.py
@@ -191,86 +191,66 @@ def crawlForum(driver):
print("Crawling the AbyssForum forum")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
i = 0
- count = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
while has_next_page:
- list = topicPages(html)
- for item in list:
- itemURL = urlparse.urljoin(baseURL, str(item))
- try:
- driver.get(itemURL)
- except:
- driver.refresh()
- savePage(driver.page_source, item)
- driver.back()
- '''
- #variable to check if there is a next page for the topic
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
+ topics = topicPages(html)
+ for topic in topics:
has_next_topic_page = True
counter = 1
+ page = topic
- # check if there is a next page for the topics
while has_next_topic_page:
- # try to access next page of th topic
- itemURL = urlparse.urljoin(baseURL, str(item))
+ itemURL = urlparse.urljoin(baseURL, str(page))
try:
driver.get(itemURL)
except:
driver.refresh()
- savePage(driver.page_source, item)
+ savePage(driver.page_source, topic + f"page{counter}")
- # if there is a next page then go and save....
- # next page in the topic?
- try:
- temp = driver.find_element(By.XPATH, '/html/body/div/div[2]/div/div[2]/div') # /html/body/div/div[2]/div/div[2]/div/
- item = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') #/html/body/div/div[2]/div/div[2]/div
+ # comment out
+ if counter == 2:
+ break
+ try:
+ temp = driver.find_element(By.XPATH, '/html/body/div[2]/div[2]/div[2]/div[3]')
+ item = temp.find_element(by=By.CLASS_NAME, value='button button-icon-only').get_attribute('href')
if item == "":
raise NoSuchElementException
- has_next_topic_page = False
- else:
- counter += 1
+ counter += 1
+
except NoSuchElementException:
has_next_topic_page = False
# end of loop
for i in range(counter):
driver.back()
- '''
+
# comment out
break
# comment out
if count == 1:
- count = 0
break
try:
link = driver.find_element(by=By.XPATH, value = '/html/body/div[2]/div[2]/div[2]/div[2]/ul/li[9]/a').get_attribute('href')
-
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -280,10 +260,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
-
input("Crawling AbyssForum forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/AnonymousMarketplace/crawler_selenium.py b/MarketPlaces/AnonymousMarketplace/crawler_selenium.py
index 70c5315..d09469f 100644
--- a/MarketPlaces/AnonymousMarketplace/crawler_selenium.py
+++ b/MarketPlaces/AnonymousMarketplace/crawler_selenium.py
@@ -202,24 +202,23 @@ def crawlForum(driver):
print("Crawling the AnonymousMarketplace market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -231,24 +230,17 @@ def crawlForum(driver):
driver.back()
# comment out
- # break
+ break
# comment out
- # if count == 20:
- # count = 0
- # break
+ if count == 1:
+ break
#left in in case site changes
try:
link = ""
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -258,9 +250,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling AnonymousMarketplace forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/Apocalypse/crawler_selenium.py b/MarketPlaces/Apocalypse/crawler_selenium.py
index 28d6a0f..134f4d8 100644
--- a/MarketPlaces/Apocalypse/crawler_selenium.py
+++ b/MarketPlaces/Apocalypse/crawler_selenium.py
@@ -216,24 +216,23 @@ def crawlForum(driver):
print("Crawling the Apocalypse market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -245,11 +244,10 @@ def crawlForum(driver):
driver.back()
# comment out
- # break
+ break
# comment out
- if count == 20:
- count = 0
+ if count == 1:
break
try:
@@ -257,12 +255,6 @@ def crawlForum(driver):
'/html/body/div[1]/div/div[2]/nav/ul/li[5]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -272,9 +264,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling Apocalypse forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/BlackPyramid/crawler_selenium.py b/MarketPlaces/BlackPyramid/crawler_selenium.py
index af1623a..5ce0101 100644
--- a/MarketPlaces/BlackPyramid/crawler_selenium.py
+++ b/MarketPlaces/BlackPyramid/crawler_selenium.py
@@ -220,26 +220,25 @@ def crawlForum(driver):
print("Crawling the BlackPyramid market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- clicker = driver.find_element(by=By.XPATH, value='/html/body/div[2]/form/nav/nav/ul/li[2]/div/a')
- clicker.click() # open tab with url
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ clicker = driver.find_element(by=By.XPATH, value='/html/body/div[2]/form/nav/nav/ul/li[2]/div/a')
+ clicker.click() # open tab with url
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -255,7 +254,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -263,12 +261,6 @@ def crawlForum(driver):
'/html/body/center/div[4]/div/div[3]/div[23]/div[2]/input[1]')
if clicker == "":
raise NoSuchElementException
- try:
- clicker.click()
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -278,9 +270,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling BlackPyramid forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/CityMarket/crawler_selenium.py b/MarketPlaces/CityMarket/crawler_selenium.py
index 1384c18..ff30bf0 100644
--- a/MarketPlaces/CityMarket/crawler_selenium.py
+++ b/MarketPlaces/CityMarket/crawler_selenium.py
@@ -221,24 +221,23 @@ def crawlForum(driver):
print("Crawling the CityMarket market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -254,7 +253,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -262,12 +260,6 @@ def crawlForum(driver):
'/html/body/div[1]/div/div[2]/nav/ul/li[5]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -277,9 +269,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling CityMarket forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/CypherMarketplace/crawler_selenium.py b/MarketPlaces/CypherMarketplace/crawler_selenium.py
index aa587c4..120ed32 100644
--- a/MarketPlaces/CypherMarketplace/crawler_selenium.py
+++ b/MarketPlaces/CypherMarketplace/crawler_selenium.py
@@ -214,24 +214,23 @@ def crawlForum(driver):
print("Crawling the CypherMarketplace market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -247,7 +246,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -256,12 +254,6 @@ def crawlForum(driver):
link = temp.find_element(by=By.TAG_NAME, value='page-link').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -271,9 +263,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling CypherMarketplace forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/DarkFox/crawler_selenium.py b/MarketPlaces/DarkFox/crawler_selenium.py
index 3967567..61927d6 100644
--- a/MarketPlaces/DarkFox/crawler_selenium.py
+++ b/MarketPlaces/DarkFox/crawler_selenium.py
@@ -239,46 +239,47 @@ def crawlForum(driver):
print("Crawling the DarkFox market")
linksToCrawl = getInterestedLinks()
- # visited = set(linksToCrawl)
- # initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
- if count >= 500:
- break
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
- itemURL = str(item)
+ itemURL = urlparse.urljoin(baseURL, str(item))
try:
driver.get(itemURL)
except:
driver.refresh()
savePage(driver.page_source, item)
driver.back()
- count += 1
+
+ # comment out
+ break
+
+ # comment out
+ if count == 0:
+ break
try:
link = driver.find_element(by=By.XPATH, value=
'/html/body/main/div/div[2]/div/div[2]/div/div/div/nav/a[2]').get_attribute('href')
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
+ if link == "":
+ raise NoSuchElementException
+ count += 1
+
except NoSuchElementException:
has_next_page = False
@@ -286,9 +287,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling BestCardingWorld forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/DarkMatter/crawler_selenium.py b/MarketPlaces/DarkMatter/crawler_selenium.py
index 21f6035..6d6986a 100644
--- a/MarketPlaces/DarkMatter/crawler_selenium.py
+++ b/MarketPlaces/DarkMatter/crawler_selenium.py
@@ -205,26 +205,24 @@ def crawlForum(driver):
print("Crawling the DarkMatter market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
- list = productPages(html)
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+ list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
try:
@@ -239,7 +237,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -248,12 +245,6 @@ def crawlForum(driver):
link = a.get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -263,9 +254,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling DarkMatter forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/DarkTor/crawler_selenium.py b/MarketPlaces/DarkTor/crawler_selenium.py
index 3c44dc7..d84de5c 100644
--- a/MarketPlaces/DarkTor/crawler_selenium.py
+++ b/MarketPlaces/DarkTor/crawler_selenium.py
@@ -201,24 +201,23 @@ def crawlForum(driver):
print("Crawling the DarkTor market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -230,23 +229,16 @@ def crawlForum(driver):
driver.back()
# comment out
- # break
+ break
# comment out
- if count == 30:
- count = 0
+ if count == 1:
break
try:
link = ""
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -256,9 +248,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling DarkTor forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/DigitalThriftShop/crawler_selenium.py b/MarketPlaces/DigitalThriftShop/crawler_selenium.py
index e602a4f..88b460f 100644
--- a/MarketPlaces/DigitalThriftShop/crawler_selenium.py
+++ b/MarketPlaces/DigitalThriftShop/crawler_selenium.py
@@ -204,24 +204,23 @@ def crawlForum(driver):
print("Crawling the DigitalThriftShop market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -237,7 +236,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -245,12 +243,6 @@ def crawlForum(driver):
'/html/body/div[1]/div[2]/div/div[2]/main/div[1]/nav/ul/li[5]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -260,9 +252,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling DigitalThriftShop forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/LionMarketplace/crawler_selenium.py b/MarketPlaces/LionMarketplace/crawler_selenium.py
index 3310aca..d969235 100644
--- a/MarketPlaces/LionMarketplace/crawler_selenium.py
+++ b/MarketPlaces/LionMarketplace/crawler_selenium.py
@@ -212,24 +212,23 @@ def crawlForum(driver):
print("Crawling the LionMarketplace market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -245,7 +244,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -253,12 +251,6 @@ def crawlForum(driver):
'/html/body/div[2]/div[2]/div/div[2]/nav/ul/li[5]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -268,9 +260,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling LionMarketplace forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/M00nkeyMarket/crawler_selenium.py b/MarketPlaces/M00nkeyMarket/crawler_selenium.py
index 2f651a5..6816cd3 100644
--- a/MarketPlaces/M00nkeyMarket/crawler_selenium.py
+++ b/MarketPlaces/M00nkeyMarket/crawler_selenium.py
@@ -217,24 +217,23 @@ def crawlForum(driver):
print("Crawling the M00nkeyMarket market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -246,24 +245,16 @@ def crawlForum(driver):
driver.back()
# comment out
- # break
+ break
# comment out
if count == 1:
- count = 0
break
try:
link = driver.find_element(by=By.LINK_TEXT, value='Next ›').get_attribute('href')
-
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -273,9 +264,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling M00nkeyMarket done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/MikesGrandStore/crawler_selenium.py b/MarketPlaces/MikesGrandStore/crawler_selenium.py
index f24dce0..bb7d1f8 100644
--- a/MarketPlaces/MikesGrandStore/crawler_selenium.py
+++ b/MarketPlaces/MikesGrandStore/crawler_selenium.py
@@ -227,24 +227,23 @@ def crawlForum(driver):
print("Crawling the MikesGrandStore market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -256,24 +255,17 @@ def crawlForum(driver):
driver.back()
# comment out
- # break
+ break
# comment out
- # if count == 1:
- # count = 0
- # break
+ if count == 1:
+ break
try:
link = driver.find_element(by=By.XPATH, value=
'/html/body/div[1]/main/div/div[1]/div/div[3]/nav/ul/li[6]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -283,9 +275,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling MikesGrandStore forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/ThiefWorld/crawler_selenium.py b/MarketPlaces/ThiefWorld/crawler_selenium.py
index 52e8f89..1111c4d 100644
--- a/MarketPlaces/ThiefWorld/crawler_selenium.py
+++ b/MarketPlaces/ThiefWorld/crawler_selenium.py
@@ -211,24 +211,23 @@ def crawlForum(driver):
print("Crawling the ThiefWorld market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -243,8 +242,7 @@ def crawlForum(driver):
break
# comment out
- if count == 20:
- count = 0
+ if count == 1:
break
try:
@@ -252,12 +250,6 @@ def crawlForum(driver):
'/html/body/div/div[1]/div/div/div[2]/div[3]/div/ul/li[13]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -267,9 +259,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling ThiefWorld forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/Tor2door/crawler_selenium.py b/MarketPlaces/Tor2door/crawler_selenium.py
index a299c71..964c574 100644
--- a/MarketPlaces/Tor2door/crawler_selenium.py
+++ b/MarketPlaces/Tor2door/crawler_selenium.py
@@ -228,25 +228,23 @@ def crawlForum(driver):
print("Crawling the Tor2door market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
i = 0
- count = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
-
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -256,12 +254,12 @@ def crawlForum(driver):
driver.refresh()
savePage(driver.page_source, item)
driver.back()
+
# comment out
break
# comment out
if count == 1:
- count = 0
break
try:
@@ -269,15 +267,8 @@ def crawlForum(driver):
'/html/body/main/div/div/div[2]/div[11]/div/nav')
a = nav.find_element(by=By.LINK_TEXT, value="›")
link = a.get_attribute('href')
-
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -287,9 +278,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling Tor2door market done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/TorBay/crawler_selenium.py b/MarketPlaces/TorBay/crawler_selenium.py
index e59f49e..0861e82 100644
--- a/MarketPlaces/TorBay/crawler_selenium.py
+++ b/MarketPlaces/TorBay/crawler_selenium.py
@@ -198,24 +198,23 @@ def crawlForum(driver):
print("Crawling the TorBay Market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -226,25 +225,18 @@ def crawlForum(driver):
savePage(driver.page_source, item)
driver.back()
- # #comment out
- # break
- #
- # # # comment out
- # if count == 1:
- # count = 0
- # break
+ # comment out
+ break
+
+ # comment out
+ if count == 1:
+ break
try:
link = driver.find_element(by=By.XPATH, value=
'/html/body/section/div/div/div[2]/div/div[2]/ul/li[3]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -254,9 +246,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling TorBay forum done sucessfully. Press ENTER to continue\n")
diff --git a/MarketPlaces/TorMarket/crawler_selenium.py b/MarketPlaces/TorMarket/crawler_selenium.py
index 35be864..0528a05 100644
--- a/MarketPlaces/TorMarket/crawler_selenium.py
+++ b/MarketPlaces/TorMarket/crawler_selenium.py
@@ -201,24 +201,23 @@ def crawlForum(driver):
print("Crawling the TorMarket market")
linksToCrawl = getInterestedLinks()
- visited = set(linksToCrawl)
- initialTime = time.time()
- count = 0
i = 0
while i < len(linksToCrawl):
link = linksToCrawl[i]
print('Crawling :', link)
try:
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
-
has_next_page = True
+ count = 0
+
while has_next_page:
+ try:
+ driver.get(link)
+ except:
+ driver.refresh()
+ html = driver.page_source
+ savePage(html, link)
+
list = productPages(html)
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
@@ -234,7 +233,6 @@ def crawlForum(driver):
# comment out
if count == 1:
- count = 0
break
try:
@@ -242,12 +240,6 @@ def crawlForum(driver):
'/html/body/div[2]/div/div/div[1]/main/nav/ul/li[5]/a').get_attribute('href')
if link == "":
raise NoSuchElementException
- try:
- driver.get(link)
- except:
- driver.refresh()
- html = driver.page_source
- savePage(html, link)
count += 1
except NoSuchElementException:
@@ -257,9 +249,6 @@ def crawlForum(driver):
print(link, e)
i += 1
- # finalTime = time.time()
- # print finalTime - initialTime
-
input("Crawling TorMarket forum done sucessfully. Press ENTER to continue\n")