Merge branch 'khoi-dev' into 'main'

Merging current progress from Khoi into main

See merge request dw9372422/dw_pipeline_test!1

Forums/CryptBB/crawler_selenium.py

@@ -28,17 +28,17 @@ baseURL = 'http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion
 
 # Opens Tor Browser, crawls the website
 def startCrawling():
-    # opentor()
+    opentor()
     forumName = getForumName()
-    # driver = getAccess()
-    #
-    # if driver != 'down':
-    #     try:
-    #         login(driver)
-    #         crawlForum(driver)
-    #     except Exception as e:
-    #         print(driver.current_url, e)
-    #     closetor(driver)
+    driver = getAccess()
+    
+    if driver != 'down':
+        try:
+            login(driver)
+            crawlForum(driver)
+        except Exception as e:
+            print(driver.current_url, e)
+        closetor(driver)
 
     new_parse(forumName, baseURL, False)
 
@@ -107,7 +107,7 @@ def login(driver):
 
 
 # Returns the name of the website
-def getForumName():
+def getForumName() -> str:
     name = 'CryptBB'
     return name
 

Forums/CryptBB/parser.py

@@ -11,6 +11,7 @@ from bs4 import BeautifulSoup
 
 # This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
 
+
 def cryptBB_description_parser(soup):
 
     # Fields to be parsed
@@ -23,7 +24,7 @@ def cryptBB_description_parser(soup):
     sign = []               # 5 all user's signature in each post (usually a standard message after the content of the post)
     post = []               # 6 all messages of each post
     feedback = []           # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format)
-    addDate = []            # 8 *all dated of each post
+    addDate = []            # 8 all dates of each post
 
     # Finding the topic (should be just one coming from the Listing Page)
 
@@ -167,7 +168,7 @@ def cryptBB_description_parser(soup):
 def cryptBB_listing_parser(soup):
 
     nm = 0              # *this variable should receive the number of topics
-    forum = "CryptBB"   # 0 *forum name
+    forum = "OnniForums"   # 0 *forum name
     board = "-1"        # 1 *board name (the previous level of the topic in the Forum categorization tree.
                         # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
     author = []         # 2 *all authors of each topic
@@ -249,4 +250,4 @@ def cryptBB_links_parser(soup):
 
         href.append(link)
 
-    return href
+    return href

Forums/HiddenAnswers/parser.py

@@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
 
 # This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
 
-def cryptBB_description_parser(soup):
+def HiddenAnswers_description_parser(soup: BeautifulSoup):
 
     # Fields to be parsed
 

Forums/Initialization/forumsList.txt

@@ -1 +1 @@
-CryptBB
+OnniForums

Forums/Initialization/forums_mining.py

@@ -95,10 +95,11 @@ if __name__ == '__main__':
     for forum in forumsList:
         forum = forum.replace('\n','')
 
-        print("Creating listing and description directories ...")
+        print("Creating listing and description directories ... for " + forum)
         createDirectory(forum)
         time.sleep(5)  # wait for directories to be created
         input("Directories created successfully. Press ENTER to continue\n")
+        
 
         if forum == "BestCardingWorld":
             crawlerBestCardingWorld()
@@ -111,6 +112,8 @@ if __name__ == '__main__':
         elif forum == "HiddenAnswers":
             crawlerHiddenAnswers()
 
+    
+    
 
 
 

Forums/Initialization/prepare_parser.py

@@ -7,6 +7,7 @@ import shutil
 from Forums.DB_Connection.db_connection import *
 from Forums.BestCardingWorld.parser import *
 from Forums.CryptBB.parser import *
+from Forums.OnniForums.parser import *
 
 from Forums.Classifier.classify_product import predict
 # from DarkWebMining_Sample.Forums.Classifier.classify_product import predict_semi
@@ -23,8 +24,8 @@ def isRussianForum(forum):
 
         iforum = iforum.replace('\n','')
         if iforum == forum:
-           result = True
-           break
+            result = True
+            break
 
     return result
 
@@ -148,6 +149,9 @@ def new_parse(forum, url, createLog):
                 rmm = bestcardingworld_description_parser(soup)
             elif forum == "CryptBB":
                 rmm = cryptBB_description_parser(soup)
+            elif forum == "OnniForums":
+                rmm = onniForums_description_parser(soup)
+            
 
             # key = u"Top:" + rmm[0].upper().strip() + u" User:" + rmm[2][0].upper().strip()
             key = u"Url:" + os.path.basename(line2).replace(".html", "")
@@ -225,6 +229,8 @@ def new_parse(forum, url, createLog):
                     rw = bestcardingworld_listing_parser(soup)
                 elif forum == "CryptBB":
                     rw = cryptBB_listing_parser(soup)
+                elif forum == "OnniForums":
+                    rw = onniForums_listing_parser(soup)
 
             except:
 
@@ -246,9 +252,11 @@ def new_parse(forum, url, createLog):
                 for rec in rw:
 
                     rec = rec.split(',')
+                    # print(rec)
 
                     # key = u"Top:" + rec[1].upper().strip() + u" User:" + rec[5].upper().strip()
                     key = u"Url:" + cleanLink(rec[6])
+                    print(key)
 
                     if key in detPage: