khangtran
/
dark_web_forums
0
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

Merge branch 'main' into josh-dev 

# Conflicts:
#	Forums/Utilities/utilities.py
#	MarketPlaces/Initialization/geckodriver.log
#	MarketPlaces/Initialization/marketsList.txt
#	MarketPlaces/Initialization/markets_mining.py
#	MarketPlaces/Initialization/prepare_parser.py
#	MarketPlaces/Utilities/utilities.py
#	setup.ini
main
Joshua 1 year ago
parent
0baf9ff36d a2bb66165b
commit
be89dcfe30
129 changed files with 1765 additions and 54972 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -462
      Forums/AbyssForum/geckodriver.log
  2. +11
    -11
      Forums/Altenens/crawler_selenium.py
  3. +0
    -462
      Forums/Altenens/geckodriver.log
  4. BIN
      Forums/BestCardingWorld/__pycache__/__init__.cpython-311.pyc
  5. BIN
      Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-310.pyc
  6. BIN
      Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-311.pyc
  7. BIN
      Forums/BestCardingWorld/__pycache__/parser.cpython-310.pyc
  8. BIN
      Forums/BestCardingWorld/__pycache__/parser.cpython-311.pyc
  9. +0
    -462
      Forums/BestCardingWorld/geckodriver.log
  10. +14
    -11
      Forums/Cardingleaks/crawler_selenium.py
  11. +70
    -181
      Forums/Cardingleaks/parser.py
  12. BIN
      Forums/Classifier/__pycache__/__init__.cpython-310.pyc
  13. BIN
      Forums/Classifier/__pycache__/__init__.cpython-311.pyc
  14. BIN
      Forums/Classifier/__pycache__/classify_product.cpython-310.pyc
  15. BIN
      Forums/Classifier/__pycache__/classify_product.cpython-311.pyc
  16. BIN
      Forums/Classifier/__pycache__/transformer.cpython-310.pyc
  17. BIN
      Forums/Classifier/__pycache__/transformer.cpython-311.pyc
  18. +3
    -3
      Forums/CryptBB/crawler_selenium.py
  19. +0
    -462
      Forums/CryptBB/geckodriver.log
  20. BIN
      Forums/DB_Connection/__pycache__/__init__.cpython-310.pyc
  21. BIN
      Forums/DB_Connection/__pycache__/__init__.cpython-311.pyc
  22. BIN
      Forums/DB_Connection/__pycache__/db_connection.cpython-310.pyc
  23. BIN
      Forums/DB_Connection/__pycache__/db_connection.cpython-311.pyc
  24. +17
    -9
      Forums/HiddenAnswers/crawler_selenium.py
  25. +0
    -462
      Forums/HiddenAnswers/geckodriver.log
  26. +113
    -309
      Forums/HiddenAnswers/parser.py
  27. +1
    -1
      Forums/Initialization/forumsList.txt
  28. +3
    -3
      Forums/Initialization/forums_mining.py
  29. +0
    -11277
      Forums/Initialization/geckodriver.log
  30. +10
    -0
      Forums/Initialization/prepare_parser.py
  31. +9
    -9
      Forums/Libre/crawler_selenium.py
  32. +136
    -153
      Forums/Libre/parser.py
  33. +0
    -462
      Forums/OnniForums/geckodriver.log
  34. +1
    -1
      Forums/Procrax/parser.py
  35. BIN
      Forums/Utilities/__pycache__/__init__.cpython-310.pyc
  36. BIN
      Forums/Utilities/__pycache__/__init__.cpython-311.pyc
  37. BIN
      Forums/Utilities/__pycache__/utilities.cpython-310.pyc
  38. BIN
      Forums/Utilities/__pycache__/utilities.cpython-311.pyc
  39. BIN
      Forums/__pycache__/__init__.cpython-310.pyc
  40. BIN
      Forums/__pycache__/__init__.cpython-311.pyc
  41. +0
    -1483
      MarketPlaces/AnonymousMarketplace/geckodriver.log
  42. BIN
      MarketPlaces/Apocalypse/__pycache__/crawler_selenium.cpython-310.pyc
  43. BIN
      MarketPlaces/Apocalypse/__pycache__/crawler_selenium.cpython-311.pyc
  44. BIN
      MarketPlaces/Apocalypse/__pycache__/parser.cpython-310.pyc
  45. BIN
      MarketPlaces/Apocalypse/__pycache__/parser.cpython-311.pyc
  46. +0
    -1483
      MarketPlaces/Apocalypse/geckodriver.log
  47. BIN
      MarketPlaces/BlackPyramid/__pycache__/crawler_selenium.cpython-310.pyc
  48. BIN
      MarketPlaces/BlackPyramid/__pycache__/parser.cpython-310.pyc
  49. +0
    -1483
      MarketPlaces/BlackPyramid/geckodriver.log
  50. BIN
      MarketPlaces/CityMarket/__pycache__/crawler_selenium.cpython-310.pyc
  51. BIN
      MarketPlaces/CityMarket/__pycache__/crawler_selenium.cpython-311.pyc
  52. BIN
      MarketPlaces/CityMarket/__pycache__/parser.cpython-310.pyc
  53. BIN
      MarketPlaces/CityMarket/__pycache__/parser.cpython-311.pyc
  54. +0
    -1483
      MarketPlaces/CityMarket/geckodriver.log
  55. BIN
      MarketPlaces/Classifier/__pycache__/__init__.cpython-310.pyc
  56. BIN
      MarketPlaces/Classifier/__pycache__/__init__.cpython-311.pyc
  57. BIN
      MarketPlaces/Classifier/__pycache__/classify_product.cpython-310.pyc
  58. BIN
      MarketPlaces/Classifier/__pycache__/classify_product.cpython-311.pyc
  59. BIN
      MarketPlaces/Classifier/__pycache__/transformer.cpython-310.pyc
  60. BIN
      MarketPlaces/Classifier/__pycache__/transformer.cpython-311.pyc
  61. +0
    -1483
      MarketPlaces/CypherMarketplace/geckodriver.log
  62. BIN
      MarketPlaces/DB_Connection/__pycache__/__init__.cpython-310.pyc
  63. BIN
      MarketPlaces/DB_Connection/__pycache__/__init__.cpython-311.pyc
  64. BIN
      MarketPlaces/DB_Connection/__pycache__/db_connection.cpython-310.pyc
  65. BIN
      MarketPlaces/DB_Connection/__pycache__/db_connection.cpython-311.pyc
  66. BIN
      MarketPlaces/DarkFox/__pycache__/crawler_selenium.cpython-310.pyc
  67. BIN
      MarketPlaces/DarkFox/__pycache__/crawler_selenium.cpython-311.pyc
  68. BIN
      MarketPlaces/DarkFox/__pycache__/parser.cpython-310.pyc
  69. BIN
      MarketPlaces/DarkFox/__pycache__/parser.cpython-311.pyc
  70. +0
    -1483
      MarketPlaces/DarkFox/geckodriver.log
  71. BIN
      MarketPlaces/DarkMatter/__pycache__/crawler_selenium.cpython-310.pyc
  72. BIN
      MarketPlaces/DarkMatter/__pycache__/crawler_selenium.cpython-311.pyc
  73. BIN
      MarketPlaces/DarkMatter/__pycache__/parser.cpython-310.pyc
  74. BIN
      MarketPlaces/DarkMatter/__pycache__/parser.cpython-311.pyc
  75. +23
    -19
      MarketPlaces/DarkMatter/crawler_selenium.py
  76. +0
    -1483
      MarketPlaces/DarkMatter/geckodriver.log
  77. +166
    -112
      MarketPlaces/DarkMatter/parser.py
  78. BIN
      MarketPlaces/DarkTor/__pycache__/crawler_selenium.cpython-310.pyc
  79. BIN
      MarketPlaces/DarkTor/__pycache__/crawler_selenium.cpython-311.pyc
  80. BIN
      MarketPlaces/DarkTor/__pycache__/parser.cpython-310.pyc
  81. BIN
      MarketPlaces/DarkTor/__pycache__/parser.cpython-311.pyc
  82. +0
    -1483
      MarketPlaces/DarkTor/geckodriver.log
  83. BIN
      MarketPlaces/DigitalThriftShop/__pycache__/crawler_selenium.cpython-310.pyc
  84. BIN
      MarketPlaces/DigitalThriftShop/__pycache__/crawler_selenium.cpython-311.pyc
  85. BIN
      MarketPlaces/DigitalThriftShop/__pycache__/parser.cpython-310.pyc
  86. BIN
      MarketPlaces/DigitalThriftShop/__pycache__/parser.cpython-311.pyc
  87. +15
    -13
      MarketPlaces/DigitalThriftShop/crawler_selenium.py
  88. +0
    -1483
      MarketPlaces/DigitalThriftShop/geckodriver.log
  89. +119
    -243
      MarketPlaces/DigitalThriftShop/parser.py
  90. +0
    -17090
      MarketPlaces/Initialization/geckodriver.log
  91. +3
    -0
      MarketPlaces/Initialization/markets_mining.py
  92. +30
    -5
      MarketPlaces/Initialization/prepare_parser.py
  93. BIN
      MarketPlaces/LionMarketplace/__pycache__/crawler_selenium.cpython-310.pyc
  94. BIN
      MarketPlaces/LionMarketplace/__pycache__/crawler_selenium.cpython-311.pyc
  95. BIN
      MarketPlaces/LionMarketplace/__pycache__/parser.cpython-310.pyc
  96. BIN
      MarketPlaces/LionMarketplace/__pycache__/parser.cpython-311.pyc
  97. +14
    -22
      MarketPlaces/LionMarketplace/crawler_selenium.py
  98. +0
    -1483
      MarketPlaces/LionMarketplace/geckodriver.log
  99. +233
    -238
      MarketPlaces/LionMarketplace/parser.py
  100. +12
    -12
      MarketPlaces/M00nkeyMarket/crawler_selenium.py

+ 0
- 462
Forums/AbyssForum/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

+ 11
- 11
Forums/Altenens/crawler_selenium.py View File

@ -30,18 +30,18 @@ baseURL = 'https://altenens.is/'
# Opens Tor Browser, crawls the website
def startCrawling():
# opentor()
opentor()
forumName = getForumName()
# driver = getAccess()
#
# if driver != 'down':
# try:
# login(driver)
# crawlForum(driver)
# except Exception as e:
# print(driver.current_url, e)
# closetor(driver)
#
driver = getAccess()
if driver != 'down':
try:
login(driver)
crawlForum(driver)
except Exception as e:
print(driver.current_url, e)
closetor(driver)
new_parse(forumName, baseURL, False)


+ 0
- 462
Forums/Altenens/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

BIN
Forums/BestCardingWorld/__pycache__/__init__.cpython-311.pyc View File


BIN
Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
Forums/BestCardingWorld/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
Forums/BestCardingWorld/__pycache__/parser.cpython-310.pyc View File


BIN
Forums/BestCardingWorld/__pycache__/parser.cpython-311.pyc View File


+ 0
- 462
Forums/BestCardingWorld/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

+ 14
- 11
Forums/Cardingleaks/crawler_selenium.py View File

@ -3,6 +3,9 @@ __author__ = 'DarkWeb'
'''
Cardingleaks Forum Crawler (Selenium)
Crawler updated and fixed
The site has this thing sometime whereyou'll have to look at a new post everyday. makes sure
you login first before crawling.
'''
from selenium import webdriver
@ -29,17 +32,17 @@ baseURL = 'https://cardingleaks.ws/'
# Opens Tor Browser, crawls the website
def startCrawling():
opentor()
# opentor()
forumName = getForumName()
driver = getAccess()
# driver = getAccess()
if driver != 'down':
try:
login(driver)
crawlForum(driver)
except Exception as e:
print(driver.current_url, e)
closetor(driver)
# if driver != 'down':
# try:
# login(driver)
# crawlForum(driver)
# except Exception as e:
# print(driver.current_url, e)
# closetor(driver)
new_parse(forumName, baseURL, False)
@ -242,7 +245,7 @@ def crawlForum(driver):
savePage(driver.page_source, topic + f"page{counter}") # very important
# comment out
if counter == 2:
if counter == 5:
break
try:
@ -261,7 +264,7 @@ def crawlForum(driver):
break
# comment out
if count == 1:
if count == 10:
break
try:


+ 70
- 181
Forums/Cardingleaks/parser.py View File

@ -7,12 +7,12 @@ from datetime import timedelta
import re
# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, ResultSet, Tag
# This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
def cryptBB_description_parser(soup):
def cardingleaks_description_parser(soup: Tag):
# Fields to be parsed
@ -26,135 +26,44 @@ def cryptBB_description_parser(soup):
feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format)
addDate = [] # 8 all dates of each post
# Finding the topic (should be just one coming from the Listing Page)
li = soup.find("td", {"class": "thead"}).find('strong')
topic = li.text
topic = re.sub("\[\w*\]", '', topic)
topic = topic.replace(",","")
topic = topic.replace("\n","")
topic = cleanString(topic.strip())
# Finding the repeated tag that corresponds to the listing of posts
# try:
posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all(
'div', {"class": "post"})
# For each message (post), get all the fields we are interested to:
for ipost in posts:
# Finding a first level of the HTML page
post_wrapper = ipost.find('span', {"class": "largetext"})
# Finding the author (user) of the post
author = post_wrapper.text.strip()
user.append(cleanString(author)) # Remember to clean the problematic characters
# Finding the status of the author
smalltext = ipost.find('div', {"class": "post_author"})
'''
# Testing here two possibilities to find this status and combine them
if ipost.find('div', {"class": "deleted_post_author"}):
status.append(-1)
interest.append(-1)
reputation.append(-1)
addDate.append(-1)
post.append("THIS POST HAS BEEN REMOVED!")
sign.append(-1)
feedback.append(-1)
continue
'''
# CryptBB does have membergroup and postgroup
membergroup = smalltext.find('div', {"class": "profile-rank"})
postgroup = smalltext.find('div', {"class": "postgroup"})
if membergroup != None:
membergroup = membergroup.text.strip()
if postgroup != None:
postgroup = postgroup.text.strip()
membergroup = membergroup + " - " + postgroup
else:
if postgroup != None:
membergroup = postgroup.text.strip()
else:
membergroup = "-1"
status.append(cleanString(membergroup))
# Finding the interest of the author
# CryptBB does not have blurb
blurb = smalltext.find('li', {"class": "blurb"})
if blurb != None:
blurb = blurb.text.strip()
else:
blurb = "-1"
interest.append(cleanString(blurb))
# Finding the reputation of the user
# CryptBB does have reputation
author_stats = smalltext.find('div', {"class": "author_statistics"})
karma = author_stats.find('strong')
if karma != None:
karma = karma.text
karma = karma.replace("Community Rating: ", "")
karma = karma.replace("Karma: ", "")
karma = karma.strip()
else:
karma = "-1"
reputation.append(cleanString(karma))
# Getting here another good tag to find the post date, post content and users' signature
postarea = ipost.find('div', {"class": "post_content"})
dt = postarea.find('span', {"class": "post_date"}).text
# dt = dt.strip().split()
dt = dt.strip()
day=date.today()
if "Yesterday" in dt:
yesterday = day - timedelta(days=1)
yesterday = yesterday.strftime('%m-%d-%Y')
stime = dt.replace('Yesterday,','').strip()
date_time_obj = yesterday+ ', '+stime
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p')
elif "hours ago" in dt:
day = day.strftime('%m-%d-%Y')
date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title']
date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p')
else:
date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p')
stime = date_time_obj.strftime('%b %d, %Y')
sdate = date_time_obj.strftime('%I:%M %p')
addDate.append(date_time_obj)
# Finding the post
inner = postarea.find('div', {"class": "post_body scaleimages"})
inner = inner.text.strip()
post.append(cleanString(inner))
# Finding the user's signature
# signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"})
signature = ipost.find('div', {"class": "signature scaleimages"})
if signature != None:
signature = signature.text.strip()
# print(signature)
else:
signature = "-1"
sign.append(cleanString(signature))
# As no information about user's feedback was found, just assign "-1" to the variable
li = soup.find("h1", {"class": "p-title-value"})
topic = cleanString(li.text.strip())
post_list: ResultSet[Tag] = soup.find("div", {"class": "block-body js-replyNewMessageContainer"}).find_all("article", {"data-author": True})
for ipost in post_list:
username = ipost.get('data-author')
user.append(username)
user_status = ipost.find("h5", {"class": "userTitle message-userTitle"}).text
status.append(cleanString(user_status.strip()))
user_statistics: ResultSet[Tag] = ipost.find("div", {"class": "message-userExtras"}).find_all("dl", {"class": "pairs pairs--justified"})
user_reputation = "-1"
for stat in user_statistics:
data_type = stat.find("span").get("data-original-title")
if data_type == "Points":
user_reputation = stat.find("dd").text
break
reputation.append(cleanString(user_reputation.strip()))
interest.append("-1")
sign.append("-1")
user_post = ipost.find("div", {"class": "message-content js-messageContent"}).text
post.append(cleanString(user_post.strip()))
feedback.append("-1")
datetime_text = ipost.find("ul", {"class": "message-attribution-main listInline"}).find("time").get("datetime")
datetime_obj = datetime.strptime(datetime_text, "%Y-%m-%dT%H:%M:%S%z")
addDate.append(datetime_obj)
# Populate the final variable (this should be a list with all fields scraped)
row = (topic, user, status, reputation, interest, sign, post, feedback, addDate)
@ -165,10 +74,10 @@ def cryptBB_description_parser(soup):
# This is the method to parse the Listing Pages (one page with many posts)
def cryptBB_listing_parser(soup):
def cardingleaks_listing_parser(soup: Tag):
nm = 0 # *this variable should receive the number of topics
forum = "OnniForums" # 0 *forum name
forum = "Cardingleaks" # 0 *forum name
board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree.
# For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
author = [] # 2 *all authors of each topic
@ -181,55 +90,35 @@ def cryptBB_listing_parser(soup):
# Finding the board (should be just one)
board = soup.find('span', {"class": "active"}).text
board = cleanString(board.strip())
# Finding the repeated tag that corresponds to the listing of topics
itopics = soup.find_all('tr',{"class": "inline_row"})
for itopic in itopics:
# For each topic found, the structure to get the rest of the information can be of two types. Testing all of them
# to don't miss any topic
# Adding the topic to the topic list
try:
topics = itopic.find('span', {"class": "subject_old"}).find('a').text
except:
topics = itopic.find('span', {"class": "subject_new"}).find('a').text
topics = re.sub("\[\w*\]", '', topics)
topic.append(cleanString(topics))
# Counting how many topics we have found so far
nm = len(topic)
# Adding the url to the list of urls
try:
link = itopic.find('span', {"class": "subject_old"}).find('a').get('href')
except:
link = itopic.find('span',{"class": "subject_new"}).find('a').get('href')
href.append(link)
# Finding the author of the topic
ps = itopic.find('div', {"class":"author smalltext"}).find('a').text
user = ps.strip()
author.append(cleanString(user))
# Finding the number of replies
columns = itopic.findChildren('td',recursive=False)
replies = columns[3].text
posts.append(cleanString(replies))
# Finding the number of Views
tview = columns[4].text
views.append(cleanString(tview))
# If no information about when the topic was added, just assign "-1" to the variable
addDate.append("-1")
li = soup.find("h1", {"class": "p-title-value"})
board = cleanString(li.text.strip())
thread_list: ResultSet[Tag] = soup.find("div", {"class": "structItemContainer-group js-threadList"}).find_all("div", {"data-author": True})
nm = len(thread_list)
for thread in thread_list:
thread_author = thread.get("data-author")
author.append(thread_author)
thread_topic = thread.find("div", {"class": "structItem-title"}).text
topic.append(cleanString(thread_topic.strip()))
thread_view = thread.find("dl", {"class": "pairs pairs--justified structItem-minor"}).find("dd").text
# Context text view count (i.e., 8.8K) to numerical (i.e., 8800)
if thread_view.find("K") > 0:
thread_view = str(int(float(thread_view.replace("K", "")) * 1000))
views.append(thread_view)
thread_posts = thread.find("dl", {"class": "pairs pairs--justified"}).find("dd").text
posts.append(cleanString(thread_posts.strip()))
thread_href = thread.find("div", {"class": "structItem-title"}).find("a").get("href")
href.append(thread_href)
thread_date = thread.find("li", {"class": "structItem-startDate"}).find("time").get("datetime")
datetime_obj = datetime.strptime(thread_date, "%Y-%m-%dT%H:%M:%S%z")
addDate.append(datetime_obj)
return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate)


BIN
Forums/Classifier/__pycache__/__init__.cpython-310.pyc View File


BIN
Forums/Classifier/__pycache__/__init__.cpython-311.pyc View File


BIN
Forums/Classifier/__pycache__/classify_product.cpython-310.pyc View File


BIN
Forums/Classifier/__pycache__/classify_product.cpython-311.pyc View File


BIN
Forums/Classifier/__pycache__/transformer.cpython-310.pyc View File


BIN
Forums/Classifier/__pycache__/transformer.cpython-311.pyc View File


+ 3
- 3
Forums/CryptBB/crawler_selenium.py View File

@ -273,7 +273,7 @@ def crawlForum(driver):
savePage(driver.page_source, topic + f"page{counter}") # very important
# comment out
if counter == 2:
if counter == 10:
break
try:
@ -291,10 +291,10 @@ def crawlForum(driver):
driver.back()
# comment out
break
# break
# comment out
if count == 1:
if count == 20:
break
try:


+ 0
- 462
Forums/CryptBB/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

BIN
Forums/DB_Connection/__pycache__/__init__.cpython-310.pyc View File


BIN
Forums/DB_Connection/__pycache__/__init__.cpython-311.pyc View File


BIN
Forums/DB_Connection/__pycache__/db_connection.cpython-310.pyc View File


BIN
Forums/DB_Connection/__pycache__/db_connection.cpython-311.pyc View File


+ 17
- 9
Forums/HiddenAnswers/crawler_selenium.py View File

@ -32,7 +32,7 @@ baseURL = 'http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion
def startCrawling():
opentor()
# forumName = getForumName()
driver = getAccess()
driver: webdriver.Firefox = getAccess()
if driver != 'down':
try:
@ -170,12 +170,20 @@ def getInterestedLinks():
links = []
# Hacks
links.append('http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/index.php/questions/hacking')
# links.append('http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/index.php/questions/hacking')
# links.append('http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/index.php/questions/darknet-and-tor')
# links.append('http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/index.php/questions/internet')
links.append('http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/index.php/questions/links')
return links
def crawlForum(driver):
def crawlForum(driver: webdriver.Firefox):
print("Crawling the HiddenAnswers forum")
linksToCrawl = getInterestedLinks()
@ -211,8 +219,8 @@ def crawlForum(driver):
savePage(driver.page_source, topic + f"page{counter}") # very important
# comment out
if counter == 2:
break
# if counter == 2:
# break
try:
page = "" # no next page so far may have some later on
@ -227,14 +235,14 @@ def crawlForum(driver):
driver.back()
# comment out
break
# break
# comment out
if count == 1:
break
# if count == 1:
# break
try:
link = driver.find_element(by=By.XPATH, value='/html/body/div[2]/div[2]/div/div[3]/div[3]/ul/li[7]/a').get_attribute('href')
link = driver.find_element(by=By.CLASS_NAME, value='qa-page-next').get_attribute('href')
if link == "":
raise NoSuchElementException


+ 0
- 462
Forums/HiddenAnswers/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

+ 113
- 309
Forums/HiddenAnswers/parser.py View File

@ -1,338 +1,142 @@
__author__ = 'Helium'
# Here, we are importing the auxiliary functions to clean or convert data
from typing import List
from Forums.Utilities.utilities import *
from datetime import date
from datetime import timedelta
import re
# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, ResultSet, Tag
# This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
def HiddenAnswers_description_parser(soup: BeautifulSoup):
topic: str = "-1" # topic name
user: List[str] = [] # all users of each post
addDate: List[datetime] = [] # all dated of each post
feedback: List[str] = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format)
status: List[str] = [] # all user's authority in each post such as (adm, member, dangerous)
reputation: List[str] = [] # all user's karma in each post (usually found as a number)
sign: List[str] = [] # all user's signature in each post (usually a standard message after the content of the post)
post: List[str] = [] # all messages of each post
interest: List[str] = [] # all user's interest in each post
# Fields to be parsed
topic = "-1" # topic name
user = [] # all users of each post
addDate = [] # all dated of each post
feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format)
status = [] # all user's authority in each post such as (adm, member, dangerous)
reputation = [] # all users's karma in each post (usually found as a number)
sign = [] # all user's signature in each post (usually a standard message after the content of the post)
post = [] # all messages of each post
interest = [] # all user's interest in each post
# Finding the topic (should be just one coming from the Listing Page)
li = soup.find("td", {"class": "thead"}).find('strong')
li = soup.find("h1").find("span", {"itemprop": "name"})
topic = li.text
topic = re.sub("\[\w*\]", '', topic)
topic = topic.replace(",","")
topic = topic.replace("\n","")
topic = cleanString(topic.strip())
print(topic)
# Finding the repeated tag that corresponds to the listing of posts
# posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \
# soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"})
try:
posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all(
'div', {"class": "post"})
# print(len(posts))
# For each message (post), get all the fields we are interested to:
for ipost in posts:
# Finding a first level of the HTML page
# post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"})
post_wrapper = ipost.find('span', {"class": "largetext"})
# Finding the author (user) of the post
# author = post_wrapper.find('h4')
author = post_wrapper.text.strip()
# print("author " + author)
user.append(cleanString(author)) # Remember to clean the problematic characters
# Finding the status of the author
smalltext = ipost.find('div', {"class": "post_author"})
# Testing here two possibilities to find this status and combine them
if ipost.find('div', {"class": "deleted_post_author"}):
status.append(-1)
interest.append(-1)
reputation.append(-1)
addDate.append(-1)
post.append("THIS POST HAS BEEN REMOVED!")
sign.append(-1)
feedback.append(-1)
continue
# CryptBB does have membergroup and postgroup
membergroup = smalltext.find('div', {"class": "profile-rank"})
postgroup = smalltext.find('div', {"class": "postgroup"})
if membergroup != None:
membergroup = membergroup.text.strip()
if postgroup != None:
postgroup = postgroup.text.strip()
membergroup = membergroup + " - " + postgroup
else:
if postgroup != None:
membergroup = postgroup.text.strip()
else:
membergroup = "-1"
status.append(cleanString(membergroup))
# print("status " + cleanString(membergroup))
# Finding the interest of the author
# CryptBB does not have blurb
blurb = smalltext.find('li', {"class": "blurb"})
if blurb != None:
blurb = blurb.text.strip()
else:
blurb = "-1"
interest.append(cleanString(blurb))
# Finding the reputation of the user
# CryptBB does have reputation
author_stats = smalltext.find('div', {"class": "author_statistics"})
karma = author_stats.find('strong')
if karma != None:
karma = karma.text
karma = karma.replace("Community Rating: ", "")
karma = karma.replace("Karma: ", "")
karma = karma.strip()
else:
karma = "-1"
reputation.append(cleanString(karma))
# print("karma " + cleanString(karma))
# Getting here another good tag to find the post date, post content and users' signature
postarea = ipost.find('div', {"class": "post_content"})
dt = postarea.find('span', {"class": "post_date"}).text
# dt = dt.strip().split()
dt = dt.strip()
day=date.today()
if "Yesterday" in dt:
yesterday = day - timedelta(days=1)
yesterday = yesterday.strftime('%m-%d-%Y')
stime = dt.replace('Yesterday,','').strip()
date_time_obj = yesterday+ ', '+stime
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p')
elif "hours ago" in dt:
day = day.strftime('%m-%d-%Y')
date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title']
date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p')
else:
date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p')
stime = date_time_obj.strftime('%b %d, %Y')
sdate = date_time_obj.strftime('%I:%M %p')
addDate.append(date_time_obj)
# print("date " + str(date_time_obj))
# Finding the date of the post
# date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p')
# smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\
# .find('div', {"class": "smalltext"})
# sdatetime = smalltext.text
# sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters
# sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters
# sdatetime = sdatetime.split("on: ") # Removing unnecessary characters
# sdatetime = sdatetime[1].strip()
# stime = sdatetime[:-12:-1] # Finding the time of the post
# stime = stime[::-1]
# sdate = sdatetime.replace(stime,"") # Finding the date of the post
# sdate = sdate.replace(",","")
# sdate = sdate.strip()
# Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need
# a date format here as "mm/dd/yyyy"
# addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime)
# Finding the post
inner = postarea.find('div', {"class": "post_body scaleimages"})
inner = inner.text.strip()
# print(inner)
post.append(cleanString(inner))
# Finding the users's signature
# signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"})
signature = ipost.find('div', {"class": "signature scaleimages"})
if signature != None:
signature = signature.text.strip()
# print(signature)
else:
signature = "-1"
sign.append(cleanString(signature))
# As no information about users's feedback was found, just assign "-1" to the variable
feedback.append("-1")
except:
if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ":
user.append("-1")
status.append(-1)
interest.append(-1)
reputation.append(-1)
addDate.append(-1)
post.append("NO ACCESS TO THIS PAGE!")
sign.append(-1)
feedback.append(-1)
question: Tag = soup.find("div", {"class": "qa-part-q-view"})
question_user = question.find("span", {"class": "qa-q-view-who-data"}).text
user.append(cleanString(question_user.strip()))
question_time = question.find("span", {"class": "qa-q-view-when-data"}).find("time").get("datetime")
datetime_string = question_time.split("+")[0]
datetime_obj = datetime.strptime(datetime_string, "%Y-%m-%dT%H:%M:%S")
addDate.append(datetime_obj)
question_user_status = question.find("span", {"class": "qa-q-view-who-title"}).text
status.append(cleanString(question_user_status.strip()))
question_user_karma = question.find("span", {"class": "qa-q-view-who-points-data"}).text
# Convert karma to pure numerical string
if question_user_karma.find("k") > -1:
question_user_karma = str(float(question_user_karma.replace("k", "")) * 1000)
reputation.append(cleanString(question_user_karma.strip()))
question_content = question.find("div", {"class": "qa-q-view-content qa-post-content"}).text
post.append(cleanString(question_content.strip()))
feedback.append("-1")
sign.append("-1")
interest.append("-1")
answer_list: ResultSet[Tag] = soup.find("div", {"class": "qa-a-list"}).find_all("div", {"class": "qa-a-list-item"})
for replies in answer_list:
user_name = replies.find("span", {"class", "qa-a-item-who-data"}).text
user.append(cleanString(user_name.strip()))
date_added = replies.find("span", {"class": "qa-a-item-when"}).find("time", {"itemprop": "dateCreated"}).get('datetime')
date_string = date_added.split("+")[0]
datetime_obj = datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
addDate.append(datetime_obj)
post_data = replies.find("div", {"class": "qa-a-item-content qa-post-content"}).find("div",{"itemprop":"text"}).text
post.append(cleanString(post_data.strip()))
user_reputations = replies.find("span", {"class", "qa-a-item-who-title"}).text
status.append(cleanString(user_reputations.strip()))
karma = replies.find("span", {"class": "qa-a-item-who-points-data"}).text
# Convert karma to pure numerical string
if karma.find("k") > -1:
karma = str(float(karma.replace("k", "")) * 1000)
reputation.append(cleanString(karma.strip()))
feedback.append("-1")
sign.append("-1")
interest.append("-1")
# Populate the final variable (this should be a list with all fields scraped)
row = (topic, post, user, addDate, feedback, status, reputation, sign, interest)
row = (topic, user, status, reputation, interest, sign, post, feedback, addDate)
# Sending the results
return row
# This is the method to parse the Listing Pages (one page with many posts)
def onniForums_listing_parser(soup):
board = "-1" # board name (the previous level of the topic in the Forum categorization tree.
# For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
nm = 0 # this variable should receive the number of topics
topic = [] # all topics
user = [] # all users of each topic
post = [] # number of posts of each topic
view = [] # number of views of each topic
addDate = [] # when the topic was created (difficult to find)
href = [] # this variable should receive all cleaned urls (we will use this to do the marge between
# Listing and Description pages)
# Finding the board (should be just one)
board = soup.find('span', {"class": "active"}).text
board = cleanString(board.strip())
# Finding the repeated tag that corresponds to the listing of topics
itopics = soup.find_all('tr',{"class": "inline_row"})
index = 0
for itopic in itopics:
# For each topic found, the structure to get the rest of the information can be of two types. Testing all of them
# to don't miss any topic
# Adding the topic to the topic list
try:
topics = itopic.find('span', {"class": "subject_old"}).find('a').text
except:
topics = itopic.find('span', {"class": "subject_new"}).find('a').text
topics = re.sub("\[\w*\]", '', topics)
topic.append(cleanString(topics))
# Counting how many topics we have found so far
nm = len(topic)
# Adding the url to the list of urls
try:
link = itopic.find('span', {"class": "subject_old"}).find('a').get('href')
except:
link = itopic.find('span',{"class": "subject_new"}).find('a').get('href')
link = cleanLink(link)
href.append(link)
# Finding the author of the topic
ps = itopic.find('div', {"class":"author smalltext"}).find('a').text
author = ps.strip()
user.append(cleanString(author))
# Finding the number of replies
columns = itopic.findChildren('td',recursive=False)
posts = columns[3].text
post.append(cleanString(posts))
# Finding the number of Views
tview = columns[4].text
view.append(cleanString(tview))
# If no information about when the topic was added, just assign "-1" to the variable
#dt = itopic.find('div', {"class": "responsive-hide"}).text.split('»')[1]
#dt = dt.strip()
#date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p')
#addDate.append(date_time_obj)
addDate.append("-1")
index += 1
return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href)
# if len(tag) > 0:
#
# # Finding the topic
#
# tds = tds[0].find(tag[0])
# topics = tds.text
# topics = topics.replace(u"\xbb","")
# topics = topics.strip()
# topic.append(cleanString(topics))
#
# # Counting how many topics we have found so far
#
# nm = len(topic)
#
# # Adding the url to the list of urls
#
# link = tds.findAll('a', href=True)
# link = link[0].get('href')
# link = cleanLink(link)
# href.append(link)
#
# # Finding the author of the topic
#
# ps = itopic.find('td', {"class": tag[1]}).find('p').find('a')
# if ps == None:
# ps = itopic.find('td', {"class": tag[1]}).find('p')
# ps = ps.text.replace("Started by ","")
# else:
# ps = ps.text
# author = ps.strip()
# user.append(cleanString(author))
#
# # Finding the number of replies
#
# statistics = itopic.find('td', {"class": tag[2]})
# statistics = statistics.text
# statistics = statistics.split("Replies")
# posts = statistics[0].strip()
# post.append(cleanString(posts))
#
# # Finding the number of Views
#
# views = statistics[1]
# views = views.replace("Views","")
# views = views.strip()
# view.append(cleanString(views))
#
# # As no information about when the topic was added, just assign "-1" to the variable
#
# addDate.append("-1")
#return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href)
def HiddenAnswers_listing_parser(soup: BeautifulSoup):
board = "-1" # board name (the previous level of the topic in the Forum categorization tree.
# For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
forum: str = "HiddenAnswers"
nm: int = 0 # this variable should receive the number of topics
topic: List[str] = [] # all topics
user: List[str] = [] # all users of each topic
post: List[int] = [] # number of posts of each topic
view: List[int] = [] # number of views of each topic
addDate: List[str] = [] # when the topic was created (difficult to find)
href: List[str] = [] # this variable should receive all cleaned urls (we will use this to do the merge between
# Listing and Description pages)
# Finding the board
literature = soup.find("div", {"class": "qa-main-heading"}).find("h1")
board = literature.text
queries_by_user: ResultSet[Tag] = soup.find("div", {"class": "qa-q-list"}).find_all("div", {"class": "qa-q-list-item"})
for queries in queries_by_user:
topic_of_query = queries.find("div", {"class": "qa-q-item-title"}).find("a").text
topic.append(cleanString(topic_of_query.strip()))
author = queries.find("span", {"class": "qa-q-item-who-data"}).find("a").text
user.append(cleanString(author.strip()))
num_answers = queries.find("span", {"class": "qa-a-count-data"}).text
post.append(cleanString(num_answers.strip()))
view.append("-1")
date_posted = queries.find("span", {"class": "qa-q-item-when-data"}).text
if date_posted.find("day") > 0:
datetime_obj = datetime.now() - timedelta(days=1)
else:
datetime_obj = datetime.strptime(f"{date_posted} {date.today().year}", "%b %d %Y")
addDate.append(datetime_obj)
#this link will be cleaned
listing_href = queries.find("div", {"class": "qa-q-item-title"}).find("a").get("href")
href.append(listing_href)
#need to change this method
nm = len(topic)
return organizeTopics(forum, nm, board, user, topic, view, post, href, addDate)
#need to change this method
def hiddenanswers_links_parser(soup):


+ 1
- 1
Forums/Initialization/forumsList.txt View File

@ -1 +1 @@
Procrax
HiddenAnswers

+ 3
- 3
Forums/Initialization/forums_mining.py View File

@ -99,9 +99,9 @@ if __name__ == '__main__':
forum = forum.replace('\n','')
print("Creating listing and description directories ... for " + forum)
# createDirectory(forum)
# time.sleep(5) # wait for directories to be created
# input("Directories created successfully. Press ENTER to continue\n")
createDirectory(forum)
time.sleep(5) # wait for directories to be created
input("Directories created successfully. Press ENTER to continue\n")
if forum == "BestCardingWorld":


+ 0
- 11277
Forums/Initialization/geckodriver.log
File diff suppressed because it is too large
View File


+ 10
- 0
Forums/Initialization/prepare_parser.py View File

@ -6,10 +6,12 @@ import os, re
import shutil
from Forums.DB_Connection.db_connection import *
from Forums.BestCardingWorld.parser import *
from Forums.Cardingleaks.parser import *
from Forums.CryptBB.parser import *
from Forums.OnniForums.parser import *
from Forums.Altenens.parser import *
from Forums.Procrax.parser import *
from Forums.Libre.parser import *
from Forums.Classifier.classify_product import predict
# from DarkWebMining_Sample.Forums.Classifier.classify_product import predict_semi
@ -149,6 +151,8 @@ def new_parse(forum, url, createLog):
if forum == "BestCardingWorld":
rmm = bestcardingworld_description_parser(soup)
elif forum == "Cardingleaks":
rmm = cardingleaks_description_parser(soup)
elif forum == "CryptBB":
rmm = cryptBB_description_parser(soup)
elif forum == "OnniForums":
@ -157,6 +161,8 @@ def new_parse(forum, url, createLog):
rmm = altenens_description_parser(soup)
elif forum == "Procrax":
rmm = procrax_description_parser(soup)
elif forum == "Libre":
rmm = libre_description_parser(soup)
# key = u"Top:" + rmm[0].upper().strip() + u" User:" + rmm[2][0].upper().strip()
key = u"Url:" + os.path.basename(line2).replace(".html", "")
@ -230,6 +236,8 @@ def new_parse(forum, url, createLog):
if forum == "BestCardingWorld":
rw = bestcardingworld_listing_parser(soup)
elif forum == "Cardingleaks":
rw = cardingleaks_listing_parser(soup)
elif forum == "CryptBB":
rw = cryptBB_listing_parser(soup)
elif forum == "OnniForums":
@ -238,6 +246,8 @@ def new_parse(forum, url, createLog):
rw = altenens_listing_parser(soup)
elif forum == "Procrax":
rw = procrax_listing_parser(soup)
elif forum == "Libre":
rw = libre_listing_parser(soup)
except:


+ 9
- 9
Forums/Libre/crawler_selenium.py View File

@ -28,17 +28,17 @@ baseURL = 'http://libreeunomyly6ot7kspglmbd5cvlkogib6rozy43r2glatc6rmwauqd.onion
# Opens Tor Browser, crawls the website
def startCrawling():
opentor()
# opentor()
forumName = getForumName()
driver = getAccess()
# driver = getAccess()
if driver != 'down':
try:
login(driver)
crawlForum(driver)
except Exception as e:
print(driver.current_url, e)
closetor(driver)
# if driver != 'down':
# try:
# login(driver)
# crawlForum(driver)
# except Exception as e:
# print(driver.current_url, e)
# closetor(driver)
new_parse(forumName, baseURL, False)


+ 136
- 153
Forums/Libre/parser.py View File

@ -7,155 +7,120 @@ from datetime import timedelta
import re
# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, ResultSet, Tag
# This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
def cryptBB_description_parser(soup):
def libre_description_parser(soup: Tag):
# Fields to be parsed
topic = "-1" # 0 *topic name
user = [] # 1 *all users of each post
status = [] # 2 all user's authority in each post such as (adm, member, dangerous)
reputation = [] # 3 all user's karma in each post (usually found as a number)
interest = [] # 4 all user's interest in each post
sign = [] # 5 all user's signature in each post (usually a standard message after the content of the post)
post = [] # 6 all messages of each post
feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format)
addDate = [] # 8 all dates of each post
topic = "-1" # 0 *topic name
user = [] # 1 *all users of each post
status = [] # 2 all user's authority in each post such as (adm, member, dangerous)
reputation = [] # 3 all user's karma in each post (usually found as a number)
interest = [] # 4 all user's interest in each post
sign = [] # 5 all user's signature in each post (usually a standard message after the content of the post)
post = [] # 6 all messages of each post
feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format)
addDate = [] # 8 all dates of each post
# Finding the topic (should be just one coming from the Listing Page)
li = soup.find("td", {"class": "thead"}).find('strong')
topic = li.text
topic = re.sub("\[\w*\]", '', topic)
topic = topic.replace(",","")
topic = topic.replace("\n","")
topic = cleanString(topic.strip())
topic_found = soup.find("a", {"class": "link text-xl text-zinc-300"}).text
topic = cleanString(topic_found.strip())
original_post: Tag = soup.find("div", {"class": "flex items-start"})
original_user = original_post.find("div", {"class": "info-p"}).find("a", {"class": "link"}).text
user.append(cleanString(original_user.replace("/u/", "").strip()))
original_user_statistics: ResultSet[Tag] = original_post.find("div", {"class": "info-p"}).find_all("span")
original_time = original_user_statistics[0].text[2:]
datetime_append = datetime.strptime(original_time, "%Y-%m-%d %H:%M:%S GMT")
addDate.append(datetime_append)
original_karma = original_user_statistics[1].text[2]
reputation.append(cleanString(original_karma.strip()))
original_content = soup.find("div", {"class": "content-p"}).text
post.append(cleanString(original_content.strip()))
status.append("-1")
interest.append("-1")
sign.append("-1")
feedback.append("-1")
# Finding the repeated tag that corresponds to the listing of posts
# try:
posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all(
'div', {"class": "post"})
posts: ResultSet[Tag] = soup.find_all("div", {"class": "flex items-stretch"})
# For each message (post), get all the fields we are interested to:
for ipost in posts:
# Finding a first level of the HTML page
post_wrapper = ipost.find('span', {"class": "largetext"})
# Finding the author (user) of the post
author = post_wrapper.text.strip()
user.append(cleanString(author)) # Remember to clean the problematic characters
# Finding the status of the author
smalltext = ipost.find('div', {"class": "post_author"})
'''
# Testing here two possibilities to find this status and combine them
if ipost.find('div', {"class": "deleted_post_author"}):
status.append(-1)
interest.append(-1)
reputation.append(-1)
addDate.append(-1)
post.append("THIS POST HAS BEEN REMOVED!")
sign.append(-1)
feedback.append(-1)
continue
'''
# CryptBB does have membergroup and postgroup
membergroup = smalltext.find('div', {"class": "profile-rank"})
postgroup = smalltext.find('div', {"class": "postgroup"})
if membergroup != None:
membergroup = membergroup.text.strip()
if postgroup != None:
postgroup = postgroup.text.strip()
membergroup = membergroup + " - " + postgroup
else:
if postgroup != None:
membergroup = postgroup.text.strip()
else:
membergroup = "-1"
status.append(cleanString(membergroup))
user_name = ipost.find("p", {"class": "text-zinc-400 text-justify"}).find("a", {"class": "link"}).text
user.append(cleanString(user_name.replace("/u/", "").strip())) # Remember to clean the problematic characters
status.append("-1")
# Finding the interest of the author
# CryptBB does not have blurb
blurb = smalltext.find('li', {"class": "blurb"})
if blurb != None:
blurb = blurb.text.strip()
else:
blurb = "-1"
interest.append(cleanString(blurb))
interest.append("-1")
# Finding the reputation of the user
# CryptBB does have reputation
author_stats = smalltext.find('div', {"class": "author_statistics"})
karma = author_stats.find('strong')
if karma != None:
karma = karma.text
karma = karma.replace("Community Rating: ", "")
karma = karma.replace("Karma: ", "")
karma = karma.strip()
else:
karma = "-1"
reputation.append(cleanString(karma))
karma = ipost.find("p", {"class": "text-zinc-400 text-justify"}).text
karma_cleaned = karma.split(" ")[6]
reputation.append(cleanString(karma_cleaned.strip()))
# Getting here another good tag to find the post date, post content and users' signature
postarea = ipost.find('div', {"class": "post_content"})
dt = postarea.find('span', {"class": "post_date"}).text
# dt = dt.strip().split()
dt = dt.strip()
day=date.today()
if "Yesterday" in dt:
yesterday = day - timedelta(days=1)
yesterday = yesterday.strftime('%m-%d-%Y')
stime = dt.replace('Yesterday,','').strip()
date_time_obj = yesterday+ ', '+stime
date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p')
elif "hours ago" in dt:
day = day.strftime('%m-%d-%Y')
date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title']
date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p')
else:
date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p')
stime = date_time_obj.strftime('%b %d, %Y')
sdate = date_time_obj.strftime('%I:%M %p')
addDate.append(date_time_obj)
date_posted = ipost.find("p", {"class": "text-zinc-400 text-justify"}).text
date_time_cleaned = date_posted.replace(user_name, "")[3:-12]
print(date_time_cleaned)
datetime_append = datetime.strptime(date_time_cleaned, "%Y-%m-%d %H:%M:%S GMT")
addDate.append(datetime_append)
# Finding the post
inner = postarea.find('div', {"class": "post_body scaleimages"})
inner = inner.text.strip()
post.append(cleanString(inner))
user_post = ipost.find("div", {"class": "content-c"}).text
post.append(cleanString(user_post))
# Finding the user's signature
# signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"})
signature = ipost.find('div', {"class": "signature scaleimages"})
if signature != None:
signature = signature.text.strip()
# print(signature)
else:
signature = "-1"
sign.append(cleanString(signature))
sign.append("-1")
# As no information about user's feedback was found, just assign "-1" to the variable
feedback.append("-1")
# Populate the final variable (this should be a list with all fields scraped)
# print(topic)
# print(user)
# print(status)
# print(reputation)
# print(interest)
# print(sign)
# print(post)
# print(feedback)
# print(addDate)
# print(len(user))
# print(len(status))
# print(len(reputation))
# print(len(interest))
# print(len(sign))
# print(len(feedback))
# print(len(addDate))
row = (topic, user, status, reputation, interest, sign, post, feedback, addDate)
@ -163,76 +128,94 @@ def cryptBB_description_parser(soup):
return row
# This is the method to parse the Listing Pages (one page with many posts)
def cryptBB_listing_parser(soup):
# This is the method to parse the Listing Pages (one page with many posts)
nm = 0 # *this variable should receive the number of topics
forum = "OnniForums" # 0 *forum name
board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree.
# For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
author = [] # 2 *all authors of each topic
topic = [] # 3 *all topics
views = [] # 4 number of views of each topic
posts = [] # 5 number of posts of each topic
href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between
# Listing and Description pages)
addDate = [] # 7 when the topic was created (difficult to find)
def libre_listing_parser(soup):
nm = 0 # *this variable should receive the number of topics
forum = "Libre" # 0 *forum name
board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree.
# For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
author = [] # 2 *all authors of each topic
topic = [] # 3 *all topics
views = [] # 4 number of views of each topic
posts = [] # 5 number of posts of each topic
href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between
# Listing and Description pages)
addDate = [] # 7 when the topic was created (difficult to find)
# Finding the board (should be just one)
board = soup.find('span', {"class": "active"}).text
board = soup.find('div', {"class": "title"}).find("h1").text
board = cleanString(board.strip())
# Finding the repeated tag that corresponds to the listing of topics
itopics = soup.find_all('tr',{"class": "inline_row"})
itopics = soup.find("div", {"class", "space-y-2 mt-4"}).find_all('div', {"class": "flex box"})
nm = 0
for itopic in itopics:
nm += 1
# For each topic found, the structure to get the rest of the information can be of two types. Testing all of them
# to don't miss any topic
# Adding the topic to the topic list
try:
topics = itopic.find('span', {"class": "subject_old"}).find('a').text
except:
topics = itopic.find('span', {"class": "subject_new"}).find('a').text
topics = re.sub("\[\w*\]", '', topics)
topic.append(cleanString(topics))
# Counting how many topics we have found so far
nm = len(topic)
topic_string = itopic.find("a", {"class": "link text-xl text-zinc-300"}).text
cleaned_topic_string = cleanString(topic_string.strip())
topic.append(cleaned_topic_string)
# Adding the url to the list of urls
try:
link = itopic.find('span', {"class": "subject_old"}).find('a').get('href')
except:
link = itopic.find('span',{"class": "subject_new"}).find('a').get('href')
href.append(link)
link_to_clean = itopic.find("a", {"class": "link text-xl text-zinc-300"}).get("href")
# Finding the author of the topic
ps = itopic.find('div', {"class":"author smalltext"}).find('a').text
user = ps.strip()
author.append(cleanString(user))
href.append(link_to_clean)
# Finding the number of replies
columns = itopic.findChildren('td',recursive=False)
replies = columns[3].text
# Finding the author of the topic
username_not_cleaned = itopic.find('div', {"class": "flex-grow p-2 text-justify"}).find('a').text
username_cleaned = username_not_cleaned.split("/")[-1]
author.append(cleanString(username_cleaned))
posts.append(cleanString(replies))
# Finding the number of views
num_views = itopic.find_all("div", {"class": "flex items-center"})[0].find("p").text
views.append(cleanString(num_views))
# Finding the number of Views
tview = columns[4].text
views.append(cleanString(tview))
# Finding the number of replies
num_replies = itopic.find_all("div", {"class": "flex items-center"})[1].find("p").text
posts.append(cleanString(num_replies))
# If no information about when the topic was added, just assign "-1" to the variable
addDate.append("-1")
return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate)
date_time_concatenated = itopic.find("p", {"class": "text-sm text-zinc-400 italic"}).text
date_time_cleaned = date_time_concatenated.replace(username_not_cleaned, "")
# creating the datetime object
date_time_array = date_time_cleaned[3:]
datetime_append = datetime.strptime(date_time_array, "%Y-%m-%d %H:%M:%S GMT")
addDate.append(datetime_append)
# print(forum)
# print(nm)
# print(board)
# print(author)
# print(topic)
# print(views)
# print(href)
# print(addDate)
# print(len(author))
# print(len(topic))
# print(len(views))
# print(len(href))
# print(len(addDate))
return organizeTopics(
forum=forum,
nm=nm,
board=board,
author=author,
topic=topic,
views=views,
posts=posts,
href=href,
addDate=addDate
)
def libre_links_parser(soup):
# Returning all links that should be visited by the Crawler


+ 0
- 462
Forums/OnniForums/geckodriver.log View File

@ -1,462 +0,0 @@
1655762162153 geckodriver INFO Listening on 127.0.0.1:50333
1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762166904 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
1655762168590 Marionette INFO Listening on port 50341
1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655762207624 Marionette INFO Stopped listening on port 50341
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655762990320 geckodriver INFO Listening on 127.0.0.1:50884
1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655762995244 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
1655762996997 Marionette INFO Listening on port 50890
JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763083091 Marionette INFO Stopped listening on port 50890
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763089557 geckodriver INFO Listening on 127.0.0.1:50923
1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763094208 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
1655763095918 Marionette INFO Listening on port 50929
1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763143935 Marionette INFO Stopped listening on port 50929
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763166975 geckodriver INFO Listening on 127.0.0.1:50947
1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763171719 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
1655763173307 Marionette INFO Listening on port 50953
1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763208922 Marionette INFO Stopped listening on port 50953
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655763387409 geckodriver INFO Listening on 127.0.0.1:49752
1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655763392827 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
1655763394700 Marionette INFO Listening on port 49759
1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655763430848 Marionette INFO Stopped listening on port 49759
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655765589286 geckodriver INFO Listening on 127.0.0.1:50102
1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655765622149 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
1655765623776 Marionette INFO Listening on port 50109
1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766125168 Marionette INFO Stopped listening on port 50109
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655766505099 geckodriver INFO Listening on 127.0.0.1:50156
1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655766509876 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
1655766511328 Marionette INFO Listening on port 50162
1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655766552298 Marionette INFO Stopped listening on port 50162
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
!!! error running onStopped callback: TypeError: callback is not a function
1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767657126 geckodriver INFO Listening on 127.0.0.1:50257
1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767661843 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
1655767663330 Marionette INFO Listening on port 50263
1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767668263 Marionette INFO Stopped listening on port 50263
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767689480 geckodriver INFO Listening on 127.0.0.1:50276
1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767694215 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
1655767695725 Marionette INFO Listening on port 50282
1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767701272 Marionette INFO Stopped listening on port 50282
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767791360 geckodriver INFO Listening on 127.0.0.1:50296
1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767796001 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
1655767797534 Marionette INFO Listening on port 50302
1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767802833 Marionette INFO Stopped listening on port 50302
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767902908 geckodriver INFO Listening on 127.0.0.1:50320
1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767907484 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
1655767908782 Marionette INFO Listening on port 50326
1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767916218 Marionette INFO Stopped listening on port 50326
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767958348 geckodriver INFO Listening on 127.0.0.1:50340
1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655767963032 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
1655767964591 Marionette INFO Listening on port 50346
1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655767973210 Marionette INFO Stopped listening on port 50346
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655767998084 geckodriver INFO Listening on 127.0.0.1:50358
1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768002757 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
1655768004165 Marionette INFO Listening on port 50364
1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768009787 Marionette INFO Stopped listening on port 50364
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768083966 geckodriver INFO Listening on 127.0.0.1:50386
1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768088664 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
1655768090263 Marionette INFO Listening on port 50392
1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768091655 Marionette INFO Stopped listening on port 50392
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768101201 geckodriver INFO Listening on 127.0.0.1:50404
1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
console.log: "TorSettings: loadFromPrefs()"
console.log: "TorConnect: Init"
console.log: "TorConnect: observed profile-after-change"
console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
console.log: "TorConnect: observing topic 'TorBootstrapError'"
console.log: "TorConnect: observing topic 'TorProcessExited'"
console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
console.log: "TorConnect: observing topic 'torsettings:ready'"
console.log: "TorSettings: observed profile-after-change"
1655768105853 Marionette INFO Marionette enabled
console.log: "TorConnect: will load after bootstrap => [about:blank]"
JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
console.error: "Could not load engine [email protected]: Error: Extension is invalid"
DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
1655768107431 Marionette INFO Listening on port 50410
1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session
1655768119155 Marionette INFO Stopped listening on port 50410
JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
!!! error running onStopped callback: TypeError: callback is not a function
1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64
unwatchForTabs()@TargetList.jsm:70
unwatchForTargets()@TargetList.jsm:37
destructor()@TargetList.jsm:109
stop()@CDP.jsm:104
close()@RemoteAgent.jsm:138
1655768229472 geckodriver INFO Listening on 127.0.0.1:50431
1655768233458 webdriver::server WARN Rejected request with missing Host header

+ 1
- 1
Forums/Procrax/parser.py View File

@ -110,7 +110,7 @@ def procrax_listing_parser(soup: Tag):
datetime_obj = datetime.strptime(thread_date, "%Y-%m-%dT%H:%M:%S%z")
addDate.append(datetime_obj)
thread_link = thread.find("div", {"class": "structItem-title"}).find('a').get('href')
thread_link: str = thread.find("div", {"class": "structItem-title"}).find('a').get('href')
href.append(thread_link)


BIN
Forums/Utilities/__pycache__/__init__.cpython-310.pyc View File


BIN
Forums/Utilities/__pycache__/__init__.cpython-311.pyc View File


BIN
Forums/Utilities/__pycache__/utilities.cpython-310.pyc View File


BIN
Forums/Utilities/__pycache__/utilities.cpython-311.pyc View File


BIN
Forums/__pycache__/__init__.cpython-310.pyc View File


BIN
Forums/__pycache__/__init__.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/AnonymousMarketplace/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/Apocalypse/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/Apocalypse/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/Apocalypse/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/Apocalypse/__pycache__/parser.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/Apocalypse/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/BlackPyramid/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/BlackPyramid/__pycache__/parser.cpython-310.pyc View File


+ 0
- 1483
MarketPlaces/BlackPyramid/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/CityMarket/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/CityMarket/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/CityMarket/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/CityMarket/__pycache__/parser.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/CityMarket/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/Classifier/__pycache__/__init__.cpython-310.pyc View File


BIN
MarketPlaces/Classifier/__pycache__/__init__.cpython-311.pyc View File


BIN
MarketPlaces/Classifier/__pycache__/classify_product.cpython-310.pyc View File


BIN
MarketPlaces/Classifier/__pycache__/classify_product.cpython-311.pyc View File


BIN
MarketPlaces/Classifier/__pycache__/transformer.cpython-310.pyc View File


BIN
MarketPlaces/Classifier/__pycache__/transformer.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/CypherMarketplace/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/DB_Connection/__pycache__/__init__.cpython-310.pyc View File


BIN
MarketPlaces/DB_Connection/__pycache__/__init__.cpython-311.pyc View File


BIN
MarketPlaces/DB_Connection/__pycache__/db_connection.cpython-310.pyc View File


BIN
MarketPlaces/DB_Connection/__pycache__/db_connection.cpython-311.pyc View File


BIN
MarketPlaces/DarkFox/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/DarkFox/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/DarkFox/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/DarkFox/__pycache__/parser.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/DarkFox/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/DarkMatter/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/DarkMatter/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/DarkMatter/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/DarkMatter/__pycache__/parser.cpython-311.pyc View File


+ 23
- 19
MarketPlaces/DarkMatter/crawler_selenium.py View File

@ -2,8 +2,7 @@ __author__ = 'Helium'
'''
DarkMatter Marketplace Crawler (Selenium)
website has connection issues
not working still trying to debug
Crawler works, but it slow since there is a speed check for clicking
'''
from selenium import webdriver
@ -34,7 +33,7 @@ baseURL = 'http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion
#acts like the main method for the crawler, another function at the end of this code calls this function later
def startCrawling():
opentor()
# mktName = getMKTName()
mktName = getMKTName()
driver = getAccess()
if driver != 'down':
@ -45,7 +44,7 @@ def startCrawling():
print(driver.current_url, e)
closetor(driver)
# new_parse(forumName, baseURL, False)
new_parse(mktName, baseURL, False)
# Opens Tor Browser
@ -142,7 +141,7 @@ def getAccess():
# then allows for manual solving of captcha in the terminal
#@param: current selenium web driver
def login(driver):
input("Press ENTER when CAPTCHA is completed\n")
input("Press ENTER when CAPTCHA is completed and page is loaded\n")
# wait for page to show up (This Xpath may need to change based on different seed url)
# Saves the crawled html page, makes the directory path for html pages if not made
@ -186,14 +185,16 @@ def getNameFromURL(url):
def getInterestedLinks():
links = []
# digital
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=73')
# digital fraud software
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=76')
# legit
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=78')
# # hack guides
# links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=94')
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=94')
# # services
# links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=117')
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=117')
# # software/malware
# links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=121')
links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=121')
return links
@ -226,23 +227,26 @@ def crawlForum(driver):
for item in list:
itemURL = urlparse.urljoin(baseURL, str(item))
try:
time.sleep(1.5) # to keep from detecting click speed
driver.get(itemURL)
except:
driver.refresh()
savePage(driver.page_source, item)
time.sleep(1.5)
driver.back()
# to keep from detecting click speed
# comment out
break
# comment out
if count == 1:
break
# # comment out
# break
#
# # comment out
# if count == 1:
# break
try:
nav = driver.find_element(by=By.XPATH, value='/html/body/table[1]/tbody/tr/td/form/div/div[2]/table[2]')
a = nav.find_element(by=By.LINK_TEXT, value=">")
link = a.get_attribute('href')
# nav = driver.find_element(by=By.XPATH, value='/html/body/table[1]/tbody/tr/td/form/div/div[2]/table[2]')
# a = nav.find_element(by=By.LINK_TEXT, value=">")
link = driver.find_element(by=By.LINK_TEXT, value=">").get_attribute('href')
if link == "":
raise NoSuchElementException
count += 1


+ 0
- 1483
MarketPlaces/DarkMatter/geckodriver.log
File diff suppressed because it is too large
View File


+ 166
- 112
MarketPlaces/DarkMatter/parser.py View File

@ -35,14 +35,24 @@ def darkmatter_description_parser(soup):
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
vendor = "-1" # 0 *Vendor_Name
success = "-1" # 1 Vendor_Successful_Transactions
rating_vendor = "-1" # 2 Vendor_Rating
# 0 *Vendor_Name
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[3].find('a').text
name = cleanString(temp2.strip())
except:
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[4].find('a').text
name = cleanString(temp2.strip())
except:
print("vendor")
# product name
try:
name = soup.find('head').find('title').text
name = soup.find('div', {'class', 'title-h2'}).text
name = cleanString(name.strip())
except:
print("name")
@ -55,22 +65,54 @@ def darkmatter_description_parser(soup):
except:
print("description")
#product category
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[4].find('th').text
temp2 = cleanString(temp2)
if (temp2 == "Category"):
temp2 = temp[4].find('a').text
category = cleanString(temp2.strip())
except:
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[5].find('th').text
temp2 = cleanString(temp2.strip)
if (temp2 == "Category"):
temp2 = temp[5].find('a').text
category = cleanString(temp2.strip())
except:
print('category')
# usd
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[1].find('td').text
temp2 = temp2.replace(' USD', '')
USD = cleanString(temp2)
except:
print('USD')
# 15 Product_QuantitySold
try:
temp = soup.find('table', {'class', 'vtable'})
temp = temp.findAll('tr')
temp2 = temp[5].find('th').text
temp2 = cleanString(temp2)
temp3 = temp[6].find('th').text
temp3 = cleanString(temp3)
if (temp2 == "Sold"):
temp2 = temp[5].find('td').text
sold = cleanString(temp2.strip())
elif (temp3 == "Sold"):
temp2 = temp[6].find('td').text
sold = cleanString(temp2.strip())
except:
print('sold')
CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
category = "-1" # 7 Product_Category
views = "-1" # 8 Product_Number_Of_Views
reviews = "-1" # 9 Product_Number_Of_Reviews
rating_item = "-1" # 10 Product_Rating
addDate = "-1" # 11 Product_AddedDate
BTC = "-1" # 12 Product_BTC_SellingPrice
USD = "-1" # 13 Product_USD_SellingPrice
EURO = "-1" # 14 Product_EURO_SellingPrice
sold = "-1" # 15 Product_QuantitySold
left = "-1" # 16 Product_QuantityLeft
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
# Populating the final variable (this should be a list with all fields scraped)
row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
@ -98,6 +140,7 @@ def darkmatter_listing_parser(soup):
views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews
addDate = [] # 9 Product_AddDate
rating_item = [] # 11 Product_Rating
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice
@ -111,103 +154,114 @@ def darkmatter_listing_parser(soup):
success = [] # 20 Vendor_Successful_Transactions
href = [] # 23 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card"})
names = soup.find('div', {"class": "content"}).findAll('td', {"class": "lefted", "colspan": "3"})
left = soup.find('div', {"class": "content"}).findAll('table', {"class": "vtable"})
right = soup.find('div', {"class": "content"}).findAll('td', {"class": "vtop centered"})
# vtop centered
count = 0
# Populating the Number of Products
nm = len(listing)
for a in listing:
bae = a.findAll('a', href=True)
# Adding the url to the list of urls
link = bae[0].get('href')
link = cleanLink(link)
href.append(link)
# Finding the Product
product = bae[1].find('p').text
product = product.replace('\n', ' ')
product = product.replace(",", "")
product = product.replace("...", "")
product = product.strip()
name.append(product)
bae = a.find('div', {'class': "media-content"}).find('div').find_all('div')
if len(bae) >= 5:
# Finding Prices
price = bae[0].text
ud = price.replace(" USD", " ")
# u = ud.replace("$","")
u = ud.replace(",", "")
u = u.strip()
USD.append(u)
# bc = (prc[1]).strip(' BTC')
# BTC.append(bc)
# Finding the Vendor
vendor_name = bae[1].find('a').text
vendor_name = vendor_name.replace(",", "")
vendor_name = vendor_name.strip()
vendor.append(vendor_name)
# Finding the Category
cat = bae[2].find('small').text
cat = cat.replace("Category: ", "")
cat = cat.replace(",", "")
cat = cat.strip()
category.append(cat)
# Finding Number Sold and Quantity Left
num = bae[3].text
num = num.replace("Sold: ", "")
num = num.strip()
sold.append(num)
quant = bae[4].find('small').text
quant = quant.replace("In stock: ", "")
quant = quant.strip()
qLeft.append(quant)
# Finding Successful Transactions
freq = bae[1].text
freq = freq.replace(vendor_name, "")
freq = re.sub(r'Vendor Level \d+', "", freq)
freq = freq.replace("(", "")
freq = freq.replace(")", "")
freq = freq.strip()
success.append(freq)
# Searching for CVE and MS categories
cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if not cve:
cveValue="-1"
else:
cee = " "
for idx in cve:
cee += (idx)
cee += " "
cee = cee.replace(',', ' ')
cee = cee.replace('\n', '')
cveValue=cee
CVE.append(cveValue)
ms = a.findAll(text=re.compile('MS\d{2}-\d{3}'))
if not ms:
MSValue="-1"
else:
me = " "
for im in ms:
me += (im)
me += " "
me = me.replace(',', ' ')
me = me.replace('\n', '')
MSValue=me
MS.append(MSValue)
nm = len(names)
for a in names:
# product name
try:
temp = a.find('a').text
if ("pcs x " in temp):
index = temp.index("pcs x ")
result = temp[index + len("pcs x "):]
name.append(cleanString(result))
elif("pks x " in temp):
index = temp.index("pks x ")
result = temp[index + len("pks x "):]
name.append(cleanString(temp))
except Exception as e:
print("product name", e)
CVE.append("-1")
MS.append("-1")
temp2 = left[count].findAll('tr')
length_2 = len(temp2) - 1
# category
try:
temp = temp2[1].find('td').text
category.append(cleanString(temp.strip()))
except:
print('category')
describe.append("-1")
escrow.append("-1")
views.append("-1")
reviews.append("-1")
addDate.append("-1")
lastSeen.append("-1")
BTC.append("-1")
# usd
try:
temp3 = right[count*2].find('span').text
temp = temp3.replace(' USD', '')
USD.append(cleanString(temp))
except:
print('USD')
EURO.append("-1")
# 14 Product_QuantitySold
try:
temp3 = temp2[length_2].find('th').text
temp3 = cleanString(temp3)
if (temp3 == "Sold:"):
temp = temp2[length_2].find('td').text
sold.append(cleanString(temp.strip()))
else:
sold.append("-1")
except Exception as e:
sold.append("-1")
print('sold', e)
qLeft.append("-1")
shipFrom.append("-1")
# ship to
try:
temp3 = temp2[length_2].find('th').text
temp3 = cleanString(temp3)
if (temp3 == "Ship To:"):
temp = temp2[length_2].find('td').text
shipTo.append(cleanString(temp.strip()))
else:
shipTo.append("-1")
except Exception as e:
shipTo.append("-1")
print('shopto')
# vendor
try:
temp = temp2[0].find('a').text
vendor.append(cleanString(temp.strip()))
except:
print('vendor')
rating.append("-1")
success.append("-1")
try:
temp = a.find('a').get('href')
href.append(temp)
except:
print('href')
count += 1
rating_item.append("-1")
# Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href)
return organizeProducts(mktName, nm, vendor, rating, success, name, CVE, MS, category, describe, views,
reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href)
#called by the crawler to get description links on a listing page


BIN
MarketPlaces/DarkTor/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/DarkTor/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/DarkTor/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/DarkTor/__pycache__/parser.cpython-311.pyc View File


+ 0
- 1483
MarketPlaces/DarkTor/geckodriver.log
File diff suppressed because it is too large
View File


BIN
MarketPlaces/DigitalThriftShop/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/DigitalThriftShop/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/DigitalThriftShop/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/DigitalThriftShop/__pycache__/parser.cpython-311.pyc View File


+ 15
- 13
MarketPlaces/DigitalThriftShop/crawler_selenium.py View File

@ -21,6 +21,7 @@ import subprocess
import configparser
from bs4 import BeautifulSoup
from MarketPlaces.Initialization.prepare_parser import new_parse
from MarketPlaces.DigitalThriftShop.parser import digitalthriftshop_links_parser
from MarketPlaces.Utilities.utilities import cleanHTML
@ -31,24 +32,25 @@ baseURL = 'http://kw4zlnfhxje7top26u57iosg55i7dzuljjcyswo2clgc3mdliviswwyd.onion
# Opens Tor Browser, crawls the website, then parses, then closes tor
#acts like the main method for the crawler, another function at the end of this code calls this function later
def startCrawling():
opentor()
# mktName = getMKTName()
driver = getAccess()
# opentor()
mktName = getMKTName()
# driver = getAccess()
if driver != 'down':
try:
login(driver)
crawlForum(driver)
except Exception as e:
print(driver.current_url, e)
closetor(driver)
# if driver != 'down':
# try:
# login(driver)
# crawlForum(driver)
# except Exception as e:
# print(driver.current_url, e)
# closetor(driver)
# new_parse(forumName, baseURL, False)
new_parse(mktName, baseURL, False)
# Opens Tor Browser
#prompts for ENTER input to continue
def opentor():
from MarketPlaces.Initialization.markets_mining import config
global pid
print("Connecting Tor...")
pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path'))
@ -232,10 +234,10 @@ def crawlForum(driver):
driver.back()
# comment out
break
# break
# comment out
if count == 1:
if count == 10:
break
try:


+ 0
- 1483
MarketPlaces/DigitalThriftShop/geckodriver.log
File diff suppressed because it is too large
View File


+ 119
- 243
MarketPlaces/DigitalThriftShop/parser.py View File

@ -4,140 +4,67 @@ __author__ = 'DarkWeb'
from MarketPlaces.Utilities.utilities import *
# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, ResultSet, Tag
#parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs
#stores info it needs in different lists, these lists are returned after being organized
#@param: soup object looking at html page of description page
#return: 'row' that contains a variety of lists that each hold info on the description page
def darkfox_description_parser(soup):
def digitalThriftShop_description_parser(soup: Tag):
# Fields to be parsed
name = "-1" # 0 Product_Name
describe = "-1" # 1 Product_Description
lastSeen = "-1" # 2 Product_LastViewDate
rules = "-1" # 3 NOT USED ...
CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 5 Product_MS_Classification (Microsoft Security)
review = "-1" # 6 Product_Number_Of_Reviews
vendor = "-1" # 0 *Vendor_Name
success = "-1" # 1 Vendor_Successful_Transactions
rating_vendor = "-1" # 2 Vendor_Rating
name = "-1" # 3 *Product_Name
describe = "-1" # 4 Product_Description
CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
category = "-1" # 7 Product_Category
shipFrom = "-1" # 8 Product_ShippedFrom
shipTo = "-1" # 9 Product_ShippedTo
left = "-1" # 10 Product_QuantityLeft
escrow = "-1" # 11 Vendor_Warranty
terms = "-1" # 12 Vendor_TermsAndConditions
vendor = "-1" # 13 Vendor_Name
sold = "-1" # 14 Product_QuantitySold
addDate = "-1" # 15 Product_AddedDate
available = "-1" # 16 NOT USED ...
endDate = "-1" # 17 NOT USED ...
BTC = "-1" # 18 Product_BTC_SellingPrice
USD = "-1" # 19 Product_USD_SellingPrice
rating = "-1" # 20 Vendor_Rating
success = "-1" # 21 Vendor_Successful_Transactions
EURO = "-1" # 22 Product_EURO_SellingPrice
# Finding Product Name
name = soup.find('h1').text
name = name.replace('\n', ' ')
name = name.replace(",", "")
name = name.strip()
# Finding Vendor
vendor = soup.find('h3').find('a').text.strip()
# Finding Vendor Rating
rating = soup.find('span', {'class': "tag is-dark"}).text.strip()
# Finding Successful Transactions
success = soup.find('h3').text
success = success.replace("Vendor: ", "")
success = success.replace(vendor, "")
success = success.replace("(", "")
success = success.replace(")", "")
success = success.strip()
bae = soup.find('div', {'class': "box"}).find_all('ul')
# Finding Prices
USD = bae[1].find('strong').text.strip()
li = bae[2].find_all('li')
# Finding Escrow
escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip()
# Finding the Product Category
category = li[1].find('span', {'class': "tag is-dark"}).text.strip()
# Finding the Product Quantity Available
left = li[3].find('span', {'class': "tag is-dark"}).text.strip()
# Finding Number Sold
sold = li[4].find('span', {'class': "tag is-dark"}).text.strip()
li = bae[3].find_all('li')
# Finding Shipment Information (Origin)
if "Ships from:" in li[-2].text:
shipFrom = li[-2].text
shipFrom = shipFrom.replace("Ships from: ", "")
# shipFrom = shipFrom.replace(",", "")
shipFrom = shipFrom.strip()
# Finding Shipment Information (Destination)
shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text
shipTo = shipTo.replace("Ships to: ", "")
shipTo = shipTo.strip()
if "certain countries" in shipTo:
countries = ""
tags = li[-1].find_all('span', {'class': "tag"})
for tag in tags:
country = tag.text.strip()
countries += country + ", "
shipTo = countries.strip(", ")
# Finding the Product description
describe = soup.find('div', {'class': "pre-line"}).text
describe = describe.replace("\n", " ")
describe = describe.strip()
'''# Finding the Number of Product Reviews
tag = soup.findAll(text=re.compile('Reviews'))
for index in tag:
reviews = index
par = reviews.find('(')
if par >=0:
reviews = reviews.replace("Reviews (","")
reviews = reviews.replace(")","")
reviews = reviews.split(",")
review = str(abs(int(reviews[0])) + abs(int(reviews[1])))
else :
review = "-1"'''
# Searching for CVE and MS categories
cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if cve:
CVE = " "
for idx in cve:
CVE += (idx)
CVE += " "
CVE = CVE.replace(',', ' ')
CVE = CVE.replace('\n', '')
ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}'))
if ms:
MS = " "
for im in ms:
MS += (im)
MS += " "
MS = MS.replace(',', ' ')
MS = MS.replace('\n', '')
views = "-1" # 8 Product_Number_Of_Views
reviews = "-1" # 9 Product_Number_Of_Reviews
rating_item = "-1" # 10 Product_Rating
addDate = "-1" # 11 Product_AddedDate
BTC = "-1" # 12 Product_BTC_SellingPrice
USD = "-1" # 13 Product_USD_SellingPrice
EURO = "-1" # 14 Product_EURO_SellingPrice
sold = "-1" # 15 Product_QuantitySold
left = "-1" # 16 Product_QuantityLeft
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
product_name = soup.find("h1", {"class": "product_title entry-title"}).text
name = cleanString(product_name.strip())
product_description = soup.find("div", {"id": "tab-description"}).find("p").text
describe = cleanString(product_description.strip())
product_category = soup.find("span", {"class": "posted_in"}).find("a").text
category = cleanString(product_category.strip())
try:
product_rating: Tag = soup.find("div", {"class": "woocommerce-product-rating"})
rating_item = product_rating.find("strong", {"class": "rating"}).text
reviews = product_rating.find("span", {"Class": "rating"}).text
except Exception as e:
pass
product_BTC = soup.find("div", {"id": "price-BTC"}).find("span", {"class": "priceinfo cw-noselect"}).text
BTC = cleanString(product_BTC.strip())
product_USD = soup.find("span", {"class": "woocommerce-Price-amount amount"}).text
USD = cleanString(product_USD.replace("$", "").strip())
# Populating the final variable (this should be a list with all fields scraped)
row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor,
sold, addDate, available, endDate, BTC, USD, rating, success, EURO)
row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
BTC, USD, EURO, sold, left, shipFrom, shipTo)
# Sending the results
return row
@ -147,130 +74,79 @@ def darkfox_description_parser(soup):
#stores info it needs in different lists, these lists are returned after being organized
#@param: soup object looking at html page of listing page
#return: 'row' that contains a variety of lists that each hold info on the listing page
def darkfox_listing_parser(soup):
# Fields to be parsed
nm = 0 # Total_Products (Should be Integer)
mktName = "DarkFox" # 0 Marketplace_Name
name = [] # 1 Product_Name
CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 3 Product_MS_Classification (Microsoft Security)
category = [] # 4 Product_Category
describe = [] # 5 Product_Description
escrow = [] # 6 Vendor_Warranty
views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews
addDate = [] # 9 Product_AddDate
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice
EURO = [] # 13 Product_EURO_SellingPrice
sold = [] # 14 Product_QuantitySold
qLeft =[] # 15 Product_QuantityLeft
shipFrom = [] # 16 Product_ShippedFrom
shipTo = [] # 17 Product_ShippedTo
vendor = [] # 18 Vendor
rating = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
href = [] # 23 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card"})
# Populating the Number of Products
nm = len(listing)
for a in listing:
bae = a.findAll('a', href=True)
# Adding the url to the list of urls
link = bae[0].get('href')
link = cleanLink(link)
href.append(link)
# Finding the Product
product = bae[1].find('p').text
product = product.replace('\n', ' ')
product = product.replace(",", "")
product = product.replace("...", "")
product = product.strip()
name.append(product)
bae = a.find('div', {'class': "media-content"}).find('div').find_all('div')
if len(bae) >= 5:
# Finding Prices
price = bae[0].text
ud = price.replace(" USD", " ")
# u = ud.replace("$","")
u = ud.replace(",", "")
u = u.strip()
USD.append(u)
# bc = (prc[1]).strip(' BTC')
# BTC.append(bc)
# Finding the Vendor
vendor_name = bae[1].find('a').text
vendor_name = vendor_name.replace(",", "")
vendor_name = vendor_name.strip()
vendor.append(vendor_name)
# Finding the Category
cat = bae[2].find('small').text
cat = cat.replace("Category: ", "")
cat = cat.replace(",", "")
cat = cat.strip()
category.append(cat)
# Finding Number Sold and Quantity Left
num = bae[3].text
num = num.replace("Sold: ", "")
num = num.strip()
sold.append(num)
quant = bae[4].find('small').text
quant = quant.replace("In stock: ", "")
quant = quant.strip()
qLeft.append(quant)
# Finding Successful Transactions
freq = bae[1].text
freq = freq.replace(vendor_name, "")
freq = re.sub(r'Vendor Level \d+', "", freq)
freq = freq.replace("(", "")
freq = freq.replace(")", "")
freq = freq.strip()
success.append(freq)
# Searching for CVE and MS categories
cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if not cve:
cveValue="-1"
else:
cee = " "
for idx in cve:
cee += (idx)
cee += " "
cee = cee.replace(',', ' ')
cee = cee.replace('\n', '')
cveValue=cee
CVE.append(cveValue)
def digitalThriftShop_listing_parser(soup: Tag):
# Fields to be parsed
nm = 0 # *Total_Products (Should be Integer)
mktName = "DigitalThriftShop" # 0 *Marketplace_Name
vendor = [] # 1 *Vendor y
rating_vendor = [] # 2 Vendor_Rating
success = [] # 3 Vendor_Successful_Transactions
name = [] # 4 *Product_Name y
CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 6 Product_MS_Classification (Microsoft Security)
category = [] # 7 Product_Category y
describe = [] # 8 Product_Description
views = [] # 9 Product_Number_Of_Views
reviews = [] # 10 Product_Number_Of_Reviews
rating_item = [] # 11 Product_Rating
addDate = [] # 12 Product_AddDate
BTC = [] # 13 Product_BTC_SellingPrice
USD = [] # 14 Product_USD_SellingPrice y
EURO = [] # 15 Product_EURO_SellingPrice
sold = [] # 16 Product_QuantitySold
qLeft =[] # 17 Product_QuantityLeft
shipFrom = [] # 18 Product_ShippedFrom
shipTo = [] # 19 Product_ShippedTo
href = [] # 20 Product_Links
product_category = soup.find("h1", {"class": "woocommerce-products-header__title page-title"}).text
products_list: ResultSet[Tag] = soup.find("ul", {"class": "products columns-5"}).find_all("li")
for product in products_list:
nm += 1
vendor.append("-1")
rating_vendor.append("-1")
success.append("-1")
product_name = product.find("h2", {"class": "woocommerce-loop-product__title"}).text
name.append(cleanString(product_name.strip()))
ms = a.findAll(text=re.compile('MS\d{2}-\d{3}'))
if not ms:
MSValue="-1"
else:
me = " "
for im in ms:
me += (im)
me += " "
me = me.replace(',', ' ')
me = me.replace('\n', '')
MSValue=me
MS.append(MSValue)
CVE.append("-1")
MS.append("-1")
category.append(cleanString(product_category.strip()))
describe.append("-1")
views.append("-1")
reviews.append("-1")
try:
product_rating = product.find("div", {"class": "star-rating"}).find("strong", {"class": "rating"}).text
rating_item.append(cleanString(product_rating.strip()))
except:
rating_item.append("-1")
addDate.append("-1")
BTC.append("-1")
product_USD = product.find("span", {"class": "price"}).text
USD.append(product_USD.replace("$", "").strip())
EURO.append("-1")
sold.append("-1")
qLeft.append("-1")
shipFrom.append("-1")
shipTo.append("-1")
product_href = product.find("a", {"class": "woocommerce-LoopProduct-link woocommerce-loop-product__link"}).get("href")
href.append(cleanString(product_href.strip()))
# Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href)
return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href)
#called by the crawler to get description links on a listing page


+ 0
- 17090
MarketPlaces/Initialization/geckodriver.log
File diff suppressed because it is too large
View File


+ 3
- 0
MarketPlaces/Initialization/markets_mining.py View File

@ -20,6 +20,7 @@ from MarketPlaces.Apocalypse.crawler_selenium import crawler as crawlerApocalyps
from MarketPlaces.CityMarket.crawler_selenium import crawler as crawlerCityMarket
from MarketPlaces.DarkMatter.crawler_selenium import crawler as crawlerDarkMatter
from MarketPlaces.M00nkeyMarket.crawler_selenium import crawler as crawlerM00nkeyMarket
from MarketPlaces.ViceCity.crawler_selenium import crawler as crawlerViceCity
from MarketPlaces.HiddenMarket.crawler_selenium import crawler as crawlerHiddenMarket
import configparser
@ -107,6 +108,8 @@ if __name__ == '__main__':
crawlerDarkMatter()
elif mkt == "M00nkeyMarket":
crawlerM00nkeyMarket()
elif mkt == "ViceCity":
crawlerViceCity()
elif mkt == "HiddenMarket":
crawlerHiddenMarket()


+ 30
- 5
MarketPlaces/Initialization/prepare_parser.py View File

@ -10,8 +10,13 @@ from MarketPlaces.Tor2door.parser import *
from MarketPlaces.Apocalypse.parser import *
from MarketPlaces.ThiefWorld.parser import *
from MarketPlaces.AnonymousMarketplace.parser import *
from MarketPlaces.ViceCity.parser import *
from MarketPlaces.TorBay.parser import *
from MarketPlaces.M00nkeyMarket.parser import *
from MarketPlaces.DarkMatter.parser import *
from MarketPlaces.DigitalThriftShop.parser import *
from MarketPlaces.LionMarketplace.parser import *
from MarketPlaces.TorMarket.parser import *
from MarketPlaces.HiddenMarket.parser import *
from MarketPlaces.Classifier.classify_product import predict
@ -151,20 +156,30 @@ def new_parse(marketPlace, url, createLog):
rmm = thiefWorld_description_parser(soup)
elif marketPlace =="AnonymousMarketplace":
rmm = anonymousMarketplace_description_parser(soup)
elif marketPlace == "ViceCity":
rmm = vicecity_description_parser(soup)
elif marketPlace == "TorBay":
rmm = torbay_description_parser(soup)
elif marketPlace == "M00nkeyMarket":
rmm = m00nkey_description_parser(soup)
elif marketPlace == "HiddenMarket":
rmm = hiddenmarket_description_parser(soup)
elif marketPlace == "DarkMatter":
rmm = darkmatter_description_parser(soup)
elif marketPlace == "DigitalThriftShop":
rmm = digitalThriftShop_description_parser(soup)
elif marketPlace == "LionMarketplace":
rmm = lionmarketplace_description_parser(soup)
elif marketPlace == "TorMarket":
rmm = tormarket_description_parser(soup)
# key = u"Pr:" + rmm[0].upper()[:desc_lim1] + u" Vendor:" + rmm[13].upper()[:desc_lim2]
key = u"Url:" + os.path.basename(line2).replace(".html", "")
# save file address with description record in memory
detPage[key] = {'rmm': rmm, 'filename': os.path.basename(line2)}
except:
except Exception as e:
raise e
nError += 1
print("There was a problem to parse the file " + line2 + " in the Description section!")
@ -187,8 +202,8 @@ def new_parse(marketPlace, url, createLog):
html = open(line1.strip('\n'))
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
except Exception as e:
raise e
nError += 1
print("There was a problem to read the file " + line1 + " in the Listing section!")
if createLog:
@ -210,16 +225,26 @@ def new_parse(marketPlace, url, createLog):
rw = thiefWorld_listing_parser(soup)
elif marketPlace == "AnonymousMarketplace":
rw = anonymousMarketplace_listing_parser(soup)
elif marketPlace == "ViceCity":
rw = vicecity_listing_parser(soup)
elif marketPlace == "TorBay":
rw = torbay_listing_parser(soup)
elif marketPlace == "M00nkeyMarket":
rw = m00nkey_listing_parser(soup)
elif marketPlace == "HiddenMarket":
rw =hiddenmarket_listing_parser(soup)
elif marketPlace == "DarkMatter":
rw = darkmatter_listing_parser(soup)
elif marketPlace == "DigitalThriftShop":
rw = digitalThriftShop_listing_parser(soup)
elif marketPlace == "LionMarketplace":
rw = lionmarketplace_listing_parser(soup)
elif marketPlace == "TorMarket":
rw = tormarket_listing_parser(soup)
else:
parseError = True
except:
except Exception as e:
nError += 1
print("There was a problem to parse the file " + line1 + " in the listing section!")


BIN
MarketPlaces/LionMarketplace/__pycache__/crawler_selenium.cpython-310.pyc View File


BIN
MarketPlaces/LionMarketplace/__pycache__/crawler_selenium.cpython-311.pyc View File


BIN
MarketPlaces/LionMarketplace/__pycache__/parser.cpython-310.pyc View File


BIN
MarketPlaces/LionMarketplace/__pycache__/parser.cpython-311.pyc View File


+ 14
- 22
MarketPlaces/LionMarketplace/crawler_selenium.py View File

@ -32,7 +32,7 @@ baseURL = 'http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion
#acts like the main method for the crawler, another function at the end of this code calls this function later
def startCrawling():
opentor()
# mktName = getMKTName()
mktName = getMKTName()
driver = getAccess()
if driver != 'down':
@ -43,7 +43,7 @@ def startCrawling():
print(driver.current_url, e)
closetor(driver)
# new_parse(forumName, baseURL, False)
new_parse(mktName, baseURL, False)
# Opens Tor Browser
@ -187,20 +187,12 @@ def getInterestedLinks():
# Software/Malware
links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/16')
# # Carding
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/20')
# # Hacker for hire
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/0b19f3a0-c7e8-11ec-997b-0dcb6b05ce1d')
# # Phishing
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/18098bb0-c7e8-11ec-95e9-45b5e8898cbd')
# # Ransomware
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/ce72cee0-c7e7-11ec-a86b-c1ff2d3b2020')
# # Exploits
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/e26387c0-c7e7-11ec-a708-ab6dc5117763')
# # Spamming and Anti-Captcha
# links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/f08a9380-c7e7-11ec-918c-ffef7c670c97')
# hacked accounts
#links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/fd47b4a0-c7e7-11ec-937b-61246c4b12b3')
# Carding
links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/20')
# Hacking
links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/ba142ac0-c7e7-11ec-9bd1-fdd89c3d3f91')
# tutorial
links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/19')
return links
@ -239,12 +231,12 @@ def crawlForum(driver):
savePage(driver.page_source, item)
driver.back()
# comment out
break
# comment out
if count == 1:
break
# # comment out
# break
#
# # comment out
# if count == 1:
# break
try:
link = driver.find_element(by=By.XPATH, value=


+ 0
- 1483
MarketPlaces/LionMarketplace/geckodriver.log
File diff suppressed because it is too large
View File


+ 233
- 238
MarketPlaces/LionMarketplace/parser.py View File

@ -11,133 +11,144 @@ from bs4 import BeautifulSoup
#stores info it needs in different lists, these lists are returned after being organized
#@param: soup object looking at html page of description page
#return: 'row' that contains a variety of lists that each hold info on the description page
def darkfox_description_parser(soup):
def lionmarketplace_description_parser(soup):
# Fields to be parsed
vendor = "-1" # 0 *Vendor_Name
success = "-1" # 1 Vendor_Successful_Transactions
rating_vendor = "-1" # 2 Vendor_Rating
name = "-1" # 3 *Product_Name
describe = "-1" # 4 Product_Description
CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much
MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much
category = "-1" # 7 Product_Category
views = "-1" # 8 Product_Number_Of_Views
reviews = "-1" # 9 Product_Number_Of_Reviews
rating_item = "-1" # 10 Product_Rating
addDate = "-1" # 11 Product_AddedDate
BTC = "-1" # 12 Product_BTC_SellingPrice
USD = "-1" # 13 Product_USD_SellingPrice
EURO = "-1" # 14 Product_EURO_SellingPrice
sold = "-1" # 15 Product_QuantitySold
left = "-1" # 16 Product_QuantityLeft
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
# vendor name
try:
temp = soup.find('div', {'class': 'btn-group'}).find('a').text
vendor = (cleanString(temp.strip()))
except:
print('vendor')
vendor = "-1"
# table with info
table = soup.find('table', {'class', 'table border-0 text-left table-borderless'})
rows = table.findAll('tr')
# successful transaction
success = "-1"
# vendor rating 5
rating_vendor = '-1'
# product name
try:
temp = soup.find('div', {'class', 'row'}).find('h2').text
name = (cleanString(temp.strip()))
except:
name = '-1'
print('product name')
# product description
try:
temp = soup.find('div', {'class': "mt-4"}).findAll('p')
temp = temp[1].text
if "\n" in temp:
temp = temp.replace("\n", " ")
temp = temp.replace("\r", " ")
describe = cleanString(temp.strip())
except:
describe="-1"
print('describe')
CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much
MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much
# product category
try:
temp = rows[1].find('strong').text
category = cleanString(temp.strip())
except:
category = "-1"
print('category')
# product number of views
views = "-1"
reviews = "-1" # 9 Product_Number_Of_Reviews
rating_item = "-1" # 10 Product_Rating
addDate = "-1" # 11 Product_AddedDate
# BTC selling price box box-rounded mt-2
BTC = "-1"
# USD selling price
try:
temp = rows[2].find('strong').text
if " $" in temp:
temp = temp.replace(" $", "")
elif "$" in temp:
temp = temp.replace("$", "")
USD = cleanString((temp.strip()))
except:
try:
temp = soup.find('li').find('strong').text
if " $" in temp:
temp = temp.replace(" $", "")
elif "$" in temp:
temp = temp.replace("$", "")
USD = cleanString((temp.strip()))
except:
print("USD")
EURO = "-1" # 14 Product_EURO_SellingPrice
# product sold
try:
if (len(rows) <= 5):
temp = rows[4].find('td').text
string = cleanString(temp)
if (string == 'Left/Sold'):
temp = rows[4].findAll('td')
temp = temp[1].findAll('span')
# left
temp2 = temp[1].text
temp3 = temp[1].text
if(" items" in temp2):
temp2 = temp2.replace(" items", "")
if(" items" in temp3):
temp3 = temp3.replace(" items", "")
sold = (cleanString(temp2.strip()))
left = cleanString(temp3.strip())
else:
sold = '-1'
left = "-1"
else:
sold = '-1'
left = "-1"
except:
print("success")
sold = '-1'
left = "-1"
name = "-1" # 0 Product_Name
describe = "-1" # 1 Product_Description
lastSeen = "-1" # 2 Product_LastViewDate
rules = "-1" # 3 NOT USED ...
CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 5 Product_MS_Classification (Microsoft Security)
review = "-1" # 6 Product_Number_Of_Reviews
category = "-1" # 7 Product_Category
shipFrom = "-1" # 8 Product_ShippedFrom
shipTo = "-1" # 9 Product_ShippedTo
left = "-1" # 10 Product_QuantityLeft
escrow = "-1" # 11 Vendor_Warranty
terms = "-1" # 12 Vendor_TermsAndConditions
vendor = "-1" # 13 Vendor_Name
sold = "-1" # 14 Product_QuantitySold
addDate = "-1" # 15 Product_AddedDate
available = "-1" # 16 NOT USED ...
endDate = "-1" # 17 NOT USED ...
BTC = "-1" # 18 Product_BTC_SellingPrice
USD = "-1" # 19 Product_USD_SellingPrice
rating = "-1" # 20 Vendor_Rating
success = "-1" # 21 Vendor_Successful_Transactions
EURO = "-1" # 22 Product_EURO_SellingPrice
# Finding Product Name
name = soup.find('h1').text
name = name.replace('\n', ' ')
name = name.replace(",", "")
name = name.strip()
# Finding Vendor
vendor = soup.find('h3').find('a').text.strip()
# Finding Vendor Rating
rating = soup.find('span', {'class': "tag is-dark"}).text.strip()
# Finding Successful Transactions
success = soup.find('h3').text
success = success.replace("Vendor: ", "")
success = success.replace(vendor, "")
success = success.replace("(", "")
success = success.replace(")", "")
success = success.strip()
bae = soup.find('div', {'class': "box"}).find_all('ul')
# Finding Prices
USD = bae[1].find('strong').text.strip()
li = bae[2].find_all('li')
# Finding Escrow
escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip()
# Finding the Product Category
category = li[1].find('span', {'class': "tag is-dark"}).text.strip()
# Finding the Product Quantity Available
left = li[3].find('span', {'class': "tag is-dark"}).text.strip()
# Finding Number Sold
sold = li[4].find('span', {'class': "tag is-dark"}).text.strip()
li = bae[3].find_all('li')
# Finding Shipment Information (Origin)
if "Ships from:" in li[-2].text:
shipFrom = li[-2].text
shipFrom = shipFrom.replace("Ships from: ", "")
# shipFrom = shipFrom.replace(",", "")
shipFrom = shipFrom.strip()
# Finding Shipment Information (Destination)
shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text
shipTo = shipTo.replace("Ships to: ", "")
shipTo = shipTo.strip()
if "certain countries" in shipTo:
countries = ""
tags = li[-1].find_all('span', {'class': "tag"})
for tag in tags:
country = tag.text.strip()
countries += country + ", "
shipTo = countries.strip(", ")
# Finding the Product description
describe = soup.find('div', {'class': "pre-line"}).text
describe = describe.replace("\n", " ")
describe = describe.strip()
'''# Finding the Number of Product Reviews
tag = soup.findAll(text=re.compile('Reviews'))
for index in tag:
reviews = index
par = reviews.find('(')
if par >=0:
reviews = reviews.replace("Reviews (","")
reviews = reviews.replace(")","")
reviews = reviews.split(",")
review = str(abs(int(reviews[0])) + abs(int(reviews[1])))
else :
review = "-1"'''
# Searching for CVE and MS categories
cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if cve:
CVE = " "
for idx in cve:
CVE += (idx)
CVE += " "
CVE = CVE.replace(',', ' ')
CVE = CVE.replace('\n', '')
ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}'))
if ms:
MS = " "
for im in ms:
MS += (im)
MS += " "
MS = MS.replace(',', ' ')
MS = MS.replace('\n', '')
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
# Populating the final variable (this should be a list with all fields scraped)
row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor,
sold, addDate, available, endDate, BTC, USD, rating, success, EURO)
row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
BTC, USD, EURO, sold, left, shipFrom, shipTo)
# Sending the results
return row
@ -147,130 +158,114 @@ def darkfox_description_parser(soup):
#stores info it needs in different lists, these lists are returned after being organized
#@param: soup object looking at html page of listing page
#return: 'row' that contains a variety of lists that each hold info on the listing page
def darkfox_listing_parser(soup):
def lionmarketplace_listing_parser(soup):
# Fields to be parsed
nm = 0 # Total_Products (Should be Integer)
mktName = "DarkFox" # 0 Marketplace_Name
name = [] # 1 Product_Name
CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 3 Product_MS_Classification (Microsoft Security)
category = [] # 4 Product_Category
describe = [] # 5 Product_Description
escrow = [] # 6 Vendor_Warranty
views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews
addDate = [] # 9 Product_AddDate
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice
EURO = [] # 13 Product_EURO_SellingPrice
sold = [] # 14 Product_QuantitySold
qLeft =[] # 15 Product_QuantityLeft
shipFrom = [] # 16 Product_ShippedFrom
shipTo = [] # 17 Product_ShippedTo
vendor = [] # 18 Vendor
rating = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
href = [] # 23 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card"})
nm = 0 # *Total_Products (Should be Integer)
mktName = "M00nkeyMarket" # 0 *Marketplace_Name
vendor = [] # 1 *Vendor y
rating_vendor = [] # 2 Vendor_Rating
success = [] # 3 Vendor_Successful_Transactions
name = [] # 4 *Product_Name y
CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this
MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this
category = [] # 7 Product_Category y
describe = [] # 8 Product_Description
views = [] # 9 Product_Number_Of_Views
reviews = [] # 10 Product_Number_Of_Reviews
rating_item = [] # 11 Product_Rating
addDate = [] # 12 Product_AddDate
BTC = [] # 13 Product_BTC_SellingPrice
USD = [] # 14 Product_USD_SellingPrice y
EURO = [] # 15 Product_EURO_SellingPrice
sold = [] # 16 Product_QuantitySold
qLeft = [] # 17 Product_QuantityLeft
shipFrom = [] # 18 Product_ShippedFrom
shipTo = [] # 19 Product_ShippedTo
href = [] # 20 Product_Links
listing = soup.findAll('div', {"class": "card-body"})
# Populating the Number of Products
nm = len(listing)
for a in listing:
bae = a.findAll('a', href=True)
# Adding the url to the list of urls
link = bae[0].get('href')
link = cleanLink(link)
href.append(link)
# Finding the Product
product = bae[1].find('p').text
product = product.replace('\n', ' ')
product = product.replace(",", "")
product = product.replace("...", "")
product = product.strip()
name.append(product)
bae = a.find('div', {'class': "media-content"}).find('div').find_all('div')
if len(bae) >= 5:
# Finding Prices
price = bae[0].text
ud = price.replace(" USD", " ")
# u = ud.replace("$","")
u = ud.replace(",", "")
u = u.strip()
USD.append(u)
# bc = (prc[1]).strip(' BTC')
# BTC.append(bc)
# Finding the Vendor
vendor_name = bae[1].find('a').text
vendor_name = vendor_name.replace(",", "")
vendor_name = vendor_name.strip()
vendor.append(vendor_name)
# Finding the Category
cat = bae[2].find('small').text
cat = cat.replace("Category: ", "")
cat = cat.replace(",", "")
cat = cat.strip()
category.append(cat)
# Finding Number Sold and Quantity Left
num = bae[3].text
num = num.replace("Sold: ", "")
num = num.strip()
sold.append(num)
quant = bae[4].find('small').text
quant = quant.replace("In stock: ", "")
quant = quant.strip()
qLeft.append(quant)
# Finding Successful Transactions
freq = bae[1].text
freq = freq.replace(vendor_name, "")
freq = re.sub(r'Vendor Level \d+', "", freq)
freq = freq.replace("(", "")
freq = freq.replace(")", "")
freq = freq.strip()
success.append(freq)
# Searching for CVE and MS categories
cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if not cve:
cveValue="-1"
else:
cee = " "
for idx in cve:
cee += (idx)
cee += " "
cee = cee.replace(',', ' ')
cee = cee.replace('\n', '')
cveValue=cee
CVE.append(cveValue)
ms = a.findAll(text=re.compile('MS\d{2}-\d{3}'))
if not ms:
MSValue="-1"
else:
me = " "
for im in ms:
me += (im)
me += " "
me = me.replace(',', ' ')
me = me.replace('\n', '')
MSValue=me
MS.append(MSValue)
row = a.findAll('p')
# vendor
try:
temp = row[3].text
vendor.append(cleanString(temp.strip()))
except:
vendor.append("-1")
print('vendor')
# vendor rating
rating_vendor.append("-1")
# successful transactions CHECK AGAIN HERE
success.append("-1")
# product name
try:
temp = a.find('a').text
name.append(cleanString(temp.strip()))
except:
name.append("-1")
print('product name')
CVE.append('-1')
MS.append('-1')
# product category
try:
temp = row[2].text
if "Category: " in temp:
temp = temp.replace("Category: ", "")
category.append(cleanString(temp.strip()))
except:
print("Error in product category")
describe.append('-1')
# product views
views.append("-1")
reviews.append('-1') # 10 Product_Number_Of_Reviews
rating_item.append('-1') # 11 Product_Rating
addDate.append('-1') # 12 Product_AddDate
# BTC
BTC.append('-1')
# USD
try:
temp = row[0].find('strong').text
if ' $' in temp:
temp = temp.replace(" $", "")
USD.append(cleanString(temp.strip())) # 14 Product_USD_SellingPrice
except:
print("USD")
USD.append("-1")
EURO.append("-1") # 15 Product_EURO_SellingPrice
# product sold
sold.append("-1")
qLeft.append('-1') # 17 Product_QuantityLeft
shipFrom.append('-1') # 18 Product_ShippedFrom
shipTo.append('-1') # 19 Product_ShippedTo
# href
try:
temp = a.find('a').get('href')
href.append(temp)
except:
print('product name')
# Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href)
return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href)
#called by the crawler to get description links on a listing page


+ 12
- 12
MarketPlaces/M00nkeyMarket/crawler_selenium.py View File

@ -33,16 +33,16 @@ MARKET_NAME = 'M00nkeyMarket'
# Opens Tor Browser, crawls the website, then parses, then closes tor
#acts like the main method for the crawler, another function at the end of this code calls this function later
def startCrawling():
opentor()
driver = getAccess()
if driver != 'down':
try:
login(driver)
crawlForum(driver)
except Exception as e:
print(driver.current_url, e)
closetor(driver)
# opentor()
# driver = getAccess()
#
# if driver != 'down':
# try:
# login(driver)
# crawlForum(driver)
# except Exception as e:
# print(driver.current_url, e)
# closetor(driver)
new_parse(MARKET_NAME, BASE_URL, False)
@ -153,7 +153,7 @@ def login(driver):
# Password here
passwordBox.send_keys('genie_show_metheWorld')
input("Press ENTER when CAPTCHA and exit pressed is completed\n")
input("Press ENTER when CAPTCHA and exit pressed is completed\nWAIT FOR PAGE TO LOAD SOMETIMES THY SEND NEWSLETTERS")
# wait for listing page show up (This Xpath may need to change based on different seed url)
WebDriverWait(driver, 100).until(EC.visibility_of_element_located(
@ -203,7 +203,7 @@ def getInterestedLinks():
# software
links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=30')
# # guides
# links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
return links


Some files were not shown because too many files changed in this diff

Write Preview
Loading…
Cancel
Save