diff --git a/Forums/BestCardingWorld/__pycache__/__init__.cpython-311.pyc b/Forums/BestCardingWorld/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index d26a083..0000000 Binary files a/Forums/BestCardingWorld/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-310.pyc b/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-310.pyc deleted file mode 100644 index c058029..0000000 Binary files a/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-310.pyc and /dev/null differ diff --git a/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-311.pyc b/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-311.pyc deleted file mode 100644 index 5c8aa08..0000000 Binary files a/Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-311.pyc and /dev/null differ diff --git a/Forums/BestCardingWorld/__pycache__/parser.cpython-310.pyc b/Forums/BestCardingWorld/__pycache__/parser.cpython-310.pyc deleted file mode 100644 index 7c5c0f2..0000000 Binary files a/Forums/BestCardingWorld/__pycache__/parser.cpython-310.pyc and /dev/null differ diff --git a/Forums/BestCardingWorld/__pycache__/parser.cpython-311.pyc b/Forums/BestCardingWorld/__pycache__/parser.cpython-311.pyc deleted file mode 100644 index 82c26c2..0000000 Binary files a/Forums/BestCardingWorld/__pycache__/parser.cpython-311.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/__init__.cpython-310.pyc b/Forums/Classifier/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index adf1ecf..0000000 Binary files a/Forums/Classifier/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/__init__.cpython-311.pyc b/Forums/Classifier/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 5ab0323..0000000 Binary files a/Forums/Classifier/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/classify_product.cpython-310.pyc b/Forums/Classifier/__pycache__/classify_product.cpython-310.pyc deleted file mode 100644 index 1469aba..0000000 Binary files a/Forums/Classifier/__pycache__/classify_product.cpython-310.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/classify_product.cpython-311.pyc b/Forums/Classifier/__pycache__/classify_product.cpython-311.pyc deleted file mode 100644 index b1adb58..0000000 Binary files a/Forums/Classifier/__pycache__/classify_product.cpython-311.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/transformer.cpython-310.pyc b/Forums/Classifier/__pycache__/transformer.cpython-310.pyc deleted file mode 100644 index 748df14..0000000 Binary files a/Forums/Classifier/__pycache__/transformer.cpython-310.pyc and /dev/null differ diff --git a/Forums/Classifier/__pycache__/transformer.cpython-311.pyc b/Forums/Classifier/__pycache__/transformer.cpython-311.pyc deleted file mode 100644 index 38d1f58..0000000 Binary files a/Forums/Classifier/__pycache__/transformer.cpython-311.pyc and /dev/null differ diff --git a/Forums/DB_Connection/__pycache__/__init__.cpython-310.pyc b/Forums/DB_Connection/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index c0f2a40..0000000 Binary files a/Forums/DB_Connection/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/Forums/DB_Connection/__pycache__/__init__.cpython-311.pyc b/Forums/DB_Connection/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index b180464..0000000 Binary files a/Forums/DB_Connection/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/Forums/DB_Connection/__pycache__/db_connection.cpython-310.pyc b/Forums/DB_Connection/__pycache__/db_connection.cpython-310.pyc deleted file mode 100644 index a450752..0000000 Binary files a/Forums/DB_Connection/__pycache__/db_connection.cpython-310.pyc and /dev/null differ diff --git a/Forums/DB_Connection/__pycache__/db_connection.cpython-311.pyc b/Forums/DB_Connection/__pycache__/db_connection.cpython-311.pyc deleted file mode 100644 index eb9492d..0000000 Binary files a/Forums/DB_Connection/__pycache__/db_connection.cpython-311.pyc and /dev/null differ diff --git a/Forums/Utilities/__pycache__/__init__.cpython-310.pyc b/Forums/Utilities/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 0b13306..0000000 Binary files a/Forums/Utilities/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/Forums/Utilities/__pycache__/__init__.cpython-311.pyc b/Forums/Utilities/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 45a912d..0000000 Binary files a/Forums/Utilities/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/Forums/Utilities/__pycache__/utilities.cpython-310.pyc b/Forums/Utilities/__pycache__/utilities.cpython-310.pyc deleted file mode 100644 index 18ef440..0000000 Binary files a/Forums/Utilities/__pycache__/utilities.cpython-310.pyc and /dev/null differ diff --git a/Forums/Utilities/__pycache__/utilities.cpython-311.pyc b/Forums/Utilities/__pycache__/utilities.cpython-311.pyc deleted file mode 100644 index 1104f93..0000000 Binary files a/Forums/Utilities/__pycache__/utilities.cpython-311.pyc and /dev/null differ diff --git a/Forums/__pycache__/__init__.cpython-310.pyc b/Forums/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 3f5986c..0000000 Binary files a/Forums/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/Forums/__pycache__/__init__.cpython-311.pyc b/Forums/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index c7da357..0000000 Binary files a/Forums/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/MarketPlaces/BlackPyramid/__pycache__/crawler_selenium.cpython-310.pyc b/MarketPlaces/BlackPyramid/__pycache__/crawler_selenium.cpython-310.pyc deleted file mode 100644 index 8429502..0000000 Binary files a/MarketPlaces/BlackPyramid/__pycache__/crawler_selenium.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/BlackPyramid/__pycache__/parser.cpython-310.pyc b/MarketPlaces/BlackPyramid/__pycache__/parser.cpython-310.pyc deleted file mode 100644 index 9ea8eb5..0000000 Binary files a/MarketPlaces/BlackPyramid/__pycache__/parser.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/__init__.cpython-310.pyc b/MarketPlaces/Classifier/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 29b7be5..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/__init__.cpython-311.pyc b/MarketPlaces/Classifier/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index 4b915db..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/classify_product.cpython-310.pyc b/MarketPlaces/Classifier/__pycache__/classify_product.cpython-310.pyc deleted file mode 100644 index aa7ce4c..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/classify_product.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/classify_product.cpython-311.pyc b/MarketPlaces/Classifier/__pycache__/classify_product.cpython-311.pyc deleted file mode 100644 index b935f0d..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/classify_product.cpython-311.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/transformer.cpython-310.pyc b/MarketPlaces/Classifier/__pycache__/transformer.cpython-310.pyc deleted file mode 100644 index d0e0d65..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/transformer.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/Classifier/__pycache__/transformer.cpython-311.pyc b/MarketPlaces/Classifier/__pycache__/transformer.cpython-311.pyc deleted file mode 100644 index 1214a29..0000000 Binary files a/MarketPlaces/Classifier/__pycache__/transformer.cpython-311.pyc and /dev/null differ diff --git a/MarketPlaces/DarkMatter/crawler_selenium.py b/MarketPlaces/DarkMatter/crawler_selenium.py index afa47de..fffd3fd 100644 --- a/MarketPlaces/DarkMatter/crawler_selenium.py +++ b/MarketPlaces/DarkMatter/crawler_selenium.py @@ -32,17 +32,17 @@ baseURL = 'http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion # Opens Tor Browser, crawls the website, then parses, then closes tor #acts like the main method for the crawler, another function at the end of this code calls this function later def startCrawling(): - # opentor() + opentor() mktName = getMKTName() - # driver = getAccess() + driver = getAccess() - # if driver != 'down': - # try: - # login(driver) - # crawlForum(driver) - # except Exception as e: - # print(driver.current_url, e) - # closetor(driver) + if driver != 'down': + try: + login(driver) + crawlForum(driver) + except Exception as e: + print(driver.current_url, e) + closetor(driver) new_parse(mktName, baseURL, False) @@ -190,11 +190,11 @@ def getInterestedLinks(): # legit links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=78') # # hack guides - # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=94') + links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=94') # # services - # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=117') + links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=117') # # software/malware - # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=121') + links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=121') return links diff --git a/MarketPlaces/Initialization/marketsList.txt b/MarketPlaces/Initialization/marketsList.txt index e7488f3..032ecf3 100644 --- a/MarketPlaces/Initialization/marketsList.txt +++ b/MarketPlaces/Initialization/marketsList.txt @@ -1 +1 @@ -DigitalThriftShop \ No newline at end of file +M00nkeyMarket \ No newline at end of file diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index bea5442..9fa557c 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -224,8 +224,7 @@ def new_parse(marketPlace, url, createLog): else: parseError = True - except Exception as e:# change back - print(e) + except: nError += 1 print("There was a problem to parse the file " + line1 + " in the listing section!") @@ -268,7 +267,7 @@ def new_parse(marketPlace, url, createLog): try: persist_data(url, tuple(rec), cur) con.commit() - except Exception as e: + except: trace = traceback.format_exc() @@ -291,10 +290,7 @@ def new_parse(marketPlace, url, createLog): try: shutil.move(source, destination) num_persisted_moved += 1 - except Exception as e: - # except: - print(e) - + except: print("There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!") nError += 1 if createLog: diff --git a/MarketPlaces/M00nkeyMarket/crawler_selenium.py b/MarketPlaces/M00nkeyMarket/crawler_selenium.py index 9805f77..f0b41a4 100644 --- a/MarketPlaces/M00nkeyMarket/crawler_selenium.py +++ b/MarketPlaces/M00nkeyMarket/crawler_selenium.py @@ -153,7 +153,7 @@ def login(driver): # Password here passwordBox.send_keys('genie_show_metheWorld') - input("Press ENTER when CAPTCHA and exit pressed is completed\n") + input("Press ENTER when CAPTCHA and exit pressed is completed\nWAIT FOR PAGE TO LOAD SOMETIMES THY SEND NEWSLETTERS") # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 100).until(EC.visibility_of_element_located( @@ -203,7 +203,7 @@ def getInterestedLinks(): # software links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=30') # # guides - # links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17') + links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17') return links diff --git a/MarketPlaces/__pycache__/__init__.cpython-310.pyc b/MarketPlaces/__pycache__/__init__.cpython-310.pyc deleted file mode 100644 index 5ffd0cc..0000000 Binary files a/MarketPlaces/__pycache__/__init__.cpython-310.pyc and /dev/null differ diff --git a/MarketPlaces/__pycache__/__init__.cpython-311.pyc b/MarketPlaces/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index af5e022..0000000 Binary files a/MarketPlaces/__pycache__/__init__.cpython-311.pyc and /dev/null differ