Edited OnniForums parser to comply with database schema

1 year ago · 073473ad4b
--- a/Forums/Initialization/pycache/prepare_parser.cpython-311.pyc
+++ b/Forums/Initialization/pycache/prepare_parser.cpython-311.pyc
--- a/Forums/Initialization/prepare_parser.py
+++ b/Forums/Initialization/prepare_parser.py
@ -219,9 +219,11 @@ def new_parse(forum, url, createLog):
                for rec in rw:

                    rec = rec.split(',')
                    # print(rec)

                    # key = u"Top:" + rec[1].upper().strip() + u" User:" + rec[5].upper().strip()
                    key = u"Url:" + cleanLink(rec[6])
                    print(key)

                    if key in detPage:

--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread100KFORESUSAusers.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread100KFORESUSAusers.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread100KMexicoCCNumberNames.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread100KMexicoCCNumberNames.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread115KChinaJobDB.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread115KChinaJobDB.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread119Stealthywaystopersistmalware.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread119Stealthywaystopersistmalware.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread11MChinaLeads.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread11MChinaLeads.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread13KmegacabscomDriversDB15.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread13KmegacabscomDriversDB15.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread150KIndonesiaLeads.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread150KIndonesiaLeads.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread15KUsaBitcoinInvest2022.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread15KUsaBitcoinInvest2022.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread17Mcoinbasewalletprivatekeys.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread17Mcoinbasewalletprivatekeys.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread180KmanpowercomCandidates.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread180KmanpowercomCandidates.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread1BillionWhatsappnumbers.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread1BillionWhatsappnumbers.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread1MUSACAROWNERS.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread1MUSACAROWNERS.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread200KOGusersdatabaseleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread200KOGusersdatabaseleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread200Kenroncomemails.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread200Kenroncomemails.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread2022NVIDIAEmployeedatabase.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread2022NVIDIAEmployeedatabase.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread20kcoldwallets.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread20kcoldwallets.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread220KITALYPersonalData.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread220KITALYPersonalData.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread250kBTCWalletswithPrivateKey.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread250kBTCWalletswithPrivateKey.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread25KEPOLICEIndiaDelhiFullCourtcases.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread25KEPOLICEIndiaDelhiFullCourtcases.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread27Kkriptexorgaccounts.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread27Kkriptexorgaccounts.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread30KIndiaLeads.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread30KIndiaLeads.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread3FraudBibles.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread3FraudBibles.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread40000indatabasesthanks.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread40000indatabasesthanks.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread40KcontratanetcombrDB.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread40KcontratanetcombrDB.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500GBBreachForumsdatabaseleaked.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500GBBreachForumsdatabaseleaked.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500KChinaNickNamePhones.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500KChinaNickNamePhones.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500pxcomDBLeak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread500pxcomDBLeak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread58Kbrasiliancombolist.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread58Kbrasiliancombolist.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread600GBOnlyFansLeak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread600GBOnlyFansLeak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread650KVietnamLeads.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread650KVietnamLeads.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread65KBusinessInfoIndiaGovernmentDB.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread65KBusinessInfoIndiaGovernmentDB.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread6GB681PDFsofcybersecurityrelatedbooks.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread6GB681PDFsofcybersecurityrelatedbooks.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread70Ksahibindencomleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread70Ksahibindencomleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread70kSSNfromNormanPublicSchoolsNPS.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread70kSSNfromNormanPublicSchoolsNPS.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread90kNewZealandForex.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread90kNewZealandForex.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/Thread9KNETHERLANDPersonalDataleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/Thread9KNETHERLANDPersonalDataleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAAllinonex2.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAAllinonex2.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAJBellGreatScottishRun2023anothersportdataleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAJBellGreatScottishRun2023anothersportdataleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAUSaustraliaCampaignPersonalData.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAUSaustraliaCampaignPersonalData.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAVaversiontips.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAVaversiontips.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAboutEternelBluePostExploitation.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAboutEternelBluePostExploitation.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAcceptingCC.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAcceptingCC.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAcesstoteacheraccountinuniversitygmailandunilogin.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAcesstoteacheraccountinuniversitygmailandunilogin.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadActiveConspiracywithsmartphonesOPSECIambeingextremelycensoredbannedf.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadActiveConspiracywithsmartphonesOPSECIambeingextremelycensoredbannedf.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAgoodanonymousoffshorehost.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAgoodanonymousoffshorehost.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAirAsiacomdatabaseleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAirAsiacomdatabaseleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAlertIsitScam.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAlertIsitScam.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAllaboutSSH.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAllaboutSSH.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAmericanElectricPowerCompanyleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAmericanElectricPowerCompanyleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAninsaneamountofleakeddbssortedAZ.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAninsaneamountofleakeddbssortedAZ.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAnynewbietipsforsilentmining.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAnynewbietipsforsilentmining.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadArewesternuniontransfersscams.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadArewesternuniontransfersscams.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAttackingtheTornetworkPart0.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadAttackingtheTornetworkPart0.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBESTSELLERSPREPAIDCARS.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBESTSELLERSPREPAIDCARS.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBESTfraudbible.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBESTfraudbible.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBRAXCOMBRLEAKbrasilianindustryofsoftwaredevelopment.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBRAXCOMBRLEAKbrasilianindustryofsoftwaredevelopment.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBUNCHofRATsforFREE.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBUNCHofRATsforFREE.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBankFraudcarding.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBankFraudcarding.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBankofAmericaDatabase.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBankofAmericaDatabase.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBasiccrackingtutorial.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBasiccrackingtutorial.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBeginnerLearningCheckpointList.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBeginnerLearningCheckpointList.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestOs.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestOs.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestsiteCC.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestsiteCC.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestwaytohackuniversitiescolleges.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestwaytohackuniversitiescolleges.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestwaytoobtainCCs.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBestwaytoobtainCCs.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBigLearningCollection.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBigLearningCollection.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBinaryExploitation1BasicLinuxstackoverflows.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBinaryExploitation1BasicLinuxstackoverflows.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitLockerAbuiltindiskencryptiontoolforWindowsusers.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitLockerAbuiltindiskencryptiontoolforWindowsusers.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitchuteadecentralizedvideosharingplatformforuserswhoadvocatefreespeech.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitchuteadecentralizedvideosharingplatformforuserswhoadvocatefreespeech.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitmessageadecentralizedandsecurecommunication.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitmessageadecentralizedandsecurecommunication.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitwardenasecureandconvenientpasswordmanager.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBitwardenasecureandconvenientpasswordmanager.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlag1143TBOnlyFansLeak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlag1143TBOnlyFansLeak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlag14fullz.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlag14fullz.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagCCDumpsLeaked1642325charge.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagCCDumpsLeaked1642325charge.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagFullzLeakedTested5Charged.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagFullzLeakedTested5Charged.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagHinacomcomdataleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagHinacomcomdataleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagNigerianOilGas.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagNigerianOilGas.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagOFLeakzDumpsLatestUpdate09523.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagOFLeakzDumpsLatestUpdate09523.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagPhoeniciaUniversity.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagPhoeniciaUniversity.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagWakanintvjsondatabase.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagWakanintvjsondatabase.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagkvadroomruleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagkvadroomruleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagteleradmedicalsystemscomMEDICALleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagteleradmedicalsystemscomMEDICALleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagv3rmillionnetleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackFlagv3rmillionnetleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackflagashdodyamcoil.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBlackflagashdodyamcoil.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBooterDDoSServices.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBooterDDoSServices.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBraveAfastsecureandprivacyfriendlywebbrowser.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBraveAfastsecureandprivacyfriendlywebbrowser.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBrazilGovernmentLeak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBrazilGovernmentLeak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBureauvanDijkBVDINFO.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBureauvanDijkBVDINFO.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBurnerEmails.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBurnerEmails.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBurpSuiteProfessionalv2022127.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBurpSuiteProfessionalv2022127.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBuyingbanklogs.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBuyingbanklogs.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBypassIphones.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadBypassIphones.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadC2serverimplementationofinfostealer.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadC2serverimplementationofinfostealer.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCALAMPleak.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCALAMPleak.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCARDINGNOOB.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCARDINGNOOB.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCASHOUTPICKUP.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCASHOUTPICKUP.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCCFullzUSFRGLP3052023.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCCFullzUSFRGLP3052023.html
--- a/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCCFullzworking100good.html
+++ b/Forums/OnniForums/HTML_Pages/06272023/Description/ThreadCCFullzworking100good.html