|
@ -15,9 +15,10 @@ from MarketPlaces.M00nkeyMarket.parser import * |
|
|
from MarketPlaces.MikesGrandStore.parser import * |
|
|
from MarketPlaces.MikesGrandStore.parser import * |
|
|
from MarketPlaces.PabloEscobarMarket.parser import * |
|
|
from MarketPlaces.PabloEscobarMarket.parser import * |
|
|
from MarketPlaces.CityMarket.parser import * |
|
|
from MarketPlaces.CityMarket.parser import * |
|
|
|
|
|
from MarketPlaces.DarkBazar.parser import * |
|
|
|
|
|
from MarketPlaces.Sonanza.parser import * |
|
|
from MarketPlaces.Kingdom.parser import * |
|
|
from MarketPlaces.Kingdom.parser import * |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from MarketPlaces.Classifier.classify_product import predict |
|
|
from MarketPlaces.Classifier.classify_product import predict |
|
|
|
|
|
|
|
|
nError = 0 |
|
|
nError = 0 |
|
@ -132,6 +133,10 @@ def parse_listing(marketPlace, listingFile, soup, createLog, logFile): |
|
|
rw = pabloescobarmarket_listing_parser(soup) |
|
|
rw = pabloescobarmarket_listing_parser(soup) |
|
|
elif marketPlace == "CityMarket": |
|
|
elif marketPlace == "CityMarket": |
|
|
rw = city_listing_parser(soup) |
|
|
rw = city_listing_parser(soup) |
|
|
|
|
|
elif marketPlace == "DarkBazar": |
|
|
|
|
|
rw = darkbazar_listing_parser(soup) |
|
|
|
|
|
elif marketPlace == "Sonanza": |
|
|
|
|
|
rw = sonanza_listing_parser(soup) |
|
|
elif marketPlace == "Kingdom": |
|
|
elif marketPlace == "Kingdom": |
|
|
rw = kingdom_listing_parser(soup) |
|
|
rw = kingdom_listing_parser(soup) |
|
|
else: |
|
|
else: |
|
@ -168,6 +173,10 @@ def parse_description(marketPlace, descriptionFile, soup, createLog, logFile): |
|
|
rmm = pabloescobarmarket_description_parser(soup) |
|
|
rmm = pabloescobarmarket_description_parser(soup) |
|
|
elif marketPlace == "CityMarket": |
|
|
elif marketPlace == "CityMarket": |
|
|
rmm = city_description_parser(soup) |
|
|
rmm = city_description_parser(soup) |
|
|
|
|
|
elif marketPlace == "DarkBazar": |
|
|
|
|
|
rmm = darkbazar_description_parser(soup) |
|
|
|
|
|
elif marketPlace == "Sonanza": |
|
|
|
|
|
rmm = sonanza_description_parser(soup) |
|
|
elif marketPlace == "Kingdom": |
|
|
elif marketPlace == "Kingdom": |
|
|
rmm = kingdom_description_parser(soup) |
|
|
rmm = kingdom_description_parser(soup) |
|
|
else: |
|
|
else: |
|
@ -373,8 +382,8 @@ def new_parse(marketPlace, url, createLog): |
|
|
marketId = verifyMarketPlace(cur, marketPlace) |
|
|
marketId = verifyMarketPlace(cur, marketPlace) |
|
|
if (marketId > 0): |
|
|
if (marketId > 0): |
|
|
|
|
|
|
|
|
readListings = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing\\read", '*.htm')) |
|
|
|
|
|
readDescriptions = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description\\read", '*.htm')) |
|
|
|
|
|
|
|
|
readListings = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing\\read", '*.html')) |
|
|
|
|
|
readDescriptions = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description\\read", '*.html')) |
|
|
|
|
|
|
|
|
create_status(cur, marketId, CURRENT_DATE, len(readListings), len(readDescriptions), '1' if len(listings) > 0 else '0') |
|
|
create_status(cur, marketId, CURRENT_DATE, len(readListings), len(readDescriptions), '1' if len(listings) > 0 else '0') |
|
|
con.commit() |
|
|
con.commit() |
|
|