this is based on calsyslab project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

768 lines
28 KiB

1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
  1. __author__ = 'DarkWeb'
  2. import psycopg2
  3. import traceback
  4. from Forums.Utilities.utilities import *
  5. from dateutil.relativedelta import relativedelta, FR
  6. from scipy.spatial import distance
  7. def connectDataBase():
  8. from Forums.Initialization.forums_mining import config
  9. try:
  10. ip = config.get('PostgreSQL', 'ip')
  11. username = config.get('PostgreSQL', 'username')
  12. password = config.get('PostgreSQL', 'password')
  13. database = config.get('PostgreSQL', 'database')
  14. return psycopg2.connect(host=ip, user=username, password=password, dbname=database)
  15. except:
  16. print ("Data base " + config.get('PostgreSQL', 'database') + " not found.")
  17. raise SystemExit
  18. def verifyForum(cur, nameForum):
  19. try:
  20. cur.execute("lock table forums IN ACCESS EXCLUSIVE MODE")
  21. cur.execute("select forum_id from forums where name_forum = %(nameForum)s limit 1", {'nameForum': nameForum})
  22. recset = cur.fetchall()
  23. if recset:
  24. return recset[0]['forum_id']
  25. else:
  26. return 0
  27. except:
  28. trace = traceback.format_exc()
  29. print (trace)
  30. def verifyTopic(cur, forumId, hrefTopic):
  31. try:
  32. cur.execute("lock table topics IN ACCESS EXCLUSIVE MODE")
  33. cur.execute("select topic_id from topics where forum_id = %(forumId)s and href_topic = %(hrefTopic)s limit 1",
  34. {'forumId': forumId, 'hrefTopic': hrefTopic})
  35. recset = cur.fetchall()
  36. if recset:
  37. return recset[0]['topic_id']
  38. else:
  39. return 0
  40. except:
  41. trace = traceback.format_exc()
  42. print (trace)
  43. def verifyPost(cur, topicId, userId, dateAdded):
  44. try:
  45. cur.execute("lock table posts IN ACCESS EXCLUSIVE MODE")
  46. cur.execute("select post_id from posts where topic_id = %(topicId)s and "
  47. "user_id = %(userId)s and dateadded_post = %(dateAdded)s limit 1", {'topicId': topicId,
  48. 'userId': userId,
  49. 'dateAdded': dateAdded})
  50. recset = cur.fetchall()
  51. if recset:
  52. return recset[0]['post_id']
  53. else:
  54. return 0
  55. except:
  56. trace = traceback.format_exc()
  57. print (trace)
  58. def verifyUser(cur, nameUser, forumId):
  59. try:
  60. cur.execute("lock table users IN ACCESS EXCLUSIVE MODE")
  61. cur.execute("select user_id from users where name_user = %(nameUser)s and forum_id = %(forumId)s limit 1",
  62. {'nameUser': nameUser, 'forumId': forumId})
  63. recset = cur.fetchall()
  64. if recset:
  65. return recset[0]['user_id']
  66. else:
  67. return 0
  68. except:
  69. trace = traceback.format_exc()
  70. print (trace)
  71. def verifyImage(cur, base64Image):
  72. try:
  73. cur.execute("lock table forum_images IN ACCESS EXCLUSIVE MODE")
  74. cur.execute("select image_id from forum_images where hash_image = %(hashImage)s limit 1",
  75. {'hashImage': generate_image_hash(base64Image)})
  76. recset = cur.fetchall()
  77. if recset:
  78. return recset[0]['image_id']
  79. else:
  80. return 0
  81. except:
  82. trace = traceback.format_exc()
  83. print (trace)
  84. def getLastForum(cur):
  85. try:
  86. cur.execute("select forum_id from forums order by forum_id desc limit 1")
  87. recset = cur.fetchall()
  88. if recset:
  89. return recset[0]['forum_id']
  90. else:
  91. return 0
  92. except:
  93. trace = traceback.format_exc()
  94. print (trace)
  95. def getLastTopic(cur):
  96. try:
  97. cur.execute("select topic_id from topics order by topic_id desc limit 1")
  98. recset = cur.fetchall()
  99. if recset:
  100. return recset[0]['topic_id']
  101. else:
  102. return 0
  103. except:
  104. trace = traceback.format_exc()
  105. print (trace)
  106. def getLastUser(cur):
  107. try:
  108. cur.execute("select user_id from users order by user_id desc limit 1")
  109. recset = cur.fetchall()
  110. if recset:
  111. return recset[0]['user_id']
  112. else:
  113. return 0
  114. except:
  115. trace = traceback.format_exc()
  116. print (trace)
  117. def getLastUserVersion(cur, userId):
  118. try:
  119. cur.execute("select version_user from users_history where user_id = %(userId)s order by version_user desc limit 1", {'userId': userId})
  120. recset = cur.fetchall()
  121. if recset:
  122. return recset[0]['version_user']
  123. else:
  124. return 0
  125. except:
  126. trace = traceback.format_exc()
  127. print (trace)
  128. def getLastTopicVersion(cur, topicId):
  129. try:
  130. cur.execute("select version_topic from topics_history where topic_id = %(topicId)s order by version_topic desc limit 1", {'topicId': topicId})
  131. recset = cur.fetchall()
  132. if recset:
  133. return recset[0]['version_topic']
  134. else:
  135. return 0
  136. except:
  137. trace = traceback.format_exc()
  138. print (trace)
  139. def getLastPostVersion(cur, postId):
  140. try:
  141. cur.execute("select version_post from posts_history where post_id = %(postId)s order by version_post desc limit 1", {'postId': postId})
  142. recset = cur.fetchall()
  143. if recset:
  144. return recset[0]['version_post']
  145. else:
  146. return 0
  147. except:
  148. trace = traceback.format_exc()
  149. print (trace)
  150. def getLastPost(cur):
  151. try:
  152. cur.execute("select post_id from posts order by post_id desc limit 1")
  153. recset = cur.fetchall()
  154. if recset:
  155. return recset[0]['post_id']
  156. else:
  157. return 0
  158. except:
  159. trace = traceback.format_exc()
  160. print (trace)
  161. def getLastImage(cur):
  162. try:
  163. cur.execute("select image_id from forum_images order by image_id desc limit 1")
  164. recset = cur.fetchall()
  165. if recset:
  166. return recset[0]['image_id']
  167. else:
  168. return 0
  169. except:
  170. trace = traceback.format_exc()
  171. print (trace)
  172. def create_forum(cur, row, url):
  173. forumId = verifyForum(cur, row[0])
  174. if not forumId:
  175. forumId = int(getLastForum(cur) + 1)
  176. sql = "Insert into forums (forum_id, name_forum, url_forum, dateinserted_forum) Values (%s, %s, %s, %s)"
  177. recset = [forumId, row[0], url, row[8]]
  178. cur.execute(sql, recset)
  179. return forumId
  180. def create_topic(cur, forumId, row, authorId):
  181. hrefTopic = get_relative_url(row[6])
  182. topicId = verifyTopic(cur, forumId, hrefTopic)
  183. if not topicId:
  184. topicId = int(getLastTopic(cur) + 1)
  185. newTopic = True
  186. else:
  187. newTopic = False
  188. if newTopic:
  189. sql = "Insert into topics (topic_id, forum_id, author_id, title_topic, board_topic, views_topic, posts_topic, " \
  190. "href_topic, dateadded_topic, dateinserted_topic, classification_topic) Values (%s, %s, %s, %s, %s, %s, " \
  191. "%s, %s, %s, %s, %s)"
  192. recset = [topicId, forumId, authorId,
  193. row[3],
  194. row[1],
  195. row[4] if row[4] != '-1' else None,
  196. row[5] if row[5] != '-1' else None,
  197. hrefTopic,
  198. row[7] if row[7] != '-1' else None,
  199. row[8],
  200. row[19] if row[19] != '-1' else None]
  201. cur.execute(sql, recset)
  202. else:
  203. # Tracking potential topic changes
  204. sql = "select * from topics where topic_Id = %(topicId)s"
  205. cur.execute(sql, {'topicId': topicId})
  206. recset = cur.fetchall()
  207. if row[19] != '-1' and str(recset[0]['classification_topic']) == str(None):
  208. sql = "Update topics set classification_topic = %(classification_topic)s where topic_id = %(topicId)s"
  209. cur.execute(sql, {'classification_topic': row[19],
  210. 'topicId': topicId})
  211. elif (str(recset[0]['author_id']) != str(authorId) or
  212. str(recset[0]['title_topic']) != str(row[3]) or
  213. str(recset[0]['board_topic']) != str(row[1]) or
  214. str(recset[0]['views_topic']) != str(row[4] if row[4] != '-1' else None) or # there was a change in the topic information
  215. str(recset[0]['posts_topic']) != str(row[5] if row[5] != '-1' else None) or
  216. str(recset[0]['classification_topic']) != str(row[19] if row[19] != '-1' else recset[0]['classification_topic'])):
  217. topicVersionId = int(getLastTopicVersion(cur, topicId) + 1)
  218. sql = "Insert into topics_history (topic_id, version_topic, forum_id, author_id, title_topic, board_topic, views_topic, posts_topic, " \
  219. "href_topic, dateadded_topic, dateinserted_topic, classification_topic) Values (%s, %s, %s, %s, %s, " \
  220. "%s, %s, %s, %s, %s, %s, %s)"
  221. recset = [topicId, topicVersionId, forumId,
  222. recset[0]['author_id'],
  223. recset[0]['title_topic'],
  224. recset[0]['board_topic'],
  225. recset[0]['views_topic'],
  226. recset[0]['posts_topic'],
  227. recset[0]['href_topic'],
  228. recset[0]['dateadded_topic'],
  229. recset[0]['dateinserted_topic'],
  230. recset[0]['classification_topic']]
  231. cur.execute(sql, recset)
  232. sql = "Update topics set author_id = %(author_id)s, title_topic = %(title_topic)s, board_topic = %(board_topic)s, " \
  233. "views_topic = %(views_topic)s, posts_topic = %(posts_topic)s, dateinserted_topic = %(dateinserted_topic)s, " \
  234. "classification_topic = %(classification_topic)s where topic_id = %(topicId)s"
  235. cur.execute(sql, {'author_id': authorId,
  236. 'title_topic': row[3] if row[3] != '-1' else None,
  237. 'board_topic': row[1] if row[1] != '-1' else None,
  238. 'views_topic': row[4] if row[4] != '-1' else None,
  239. 'posts_topic': row[5] if row[5] != '-1' else None,
  240. 'dateinserted_topic': row[8],
  241. 'classification_topic': row[19] if row[19] != '-1' else None,
  242. 'topicId': topicId})
  243. return topicId
  244. def create_author(cur, row, forumId):
  245. userId = verifyUser(cur, row[2], forumId)
  246. if not userId:
  247. userId = int(getLastUser(cur) + 1)
  248. newUser = True
  249. else:
  250. newUser = False
  251. if newUser:
  252. sql = "Insert into users (user_id, forum_id, name_user, status_user, reputation_user, interest_user, " \
  253. "signature_user, image_user, dateinserted_user) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
  254. recset = [userId, forumId,
  255. row[2], 'Nan', 'Nan', 'Nan', 'Nan', None, #telling the create_user function to not track changes here
  256. row[8]]
  257. cur.execute(sql, recset)
  258. return userId
  259. def create_user(cur, row, forumId, index):
  260. userId = verifyUser(cur, row[10][index], forumId)
  261. if not userId:
  262. userId = int(getLastUser(cur) + 1)
  263. newUser = True
  264. else:
  265. newUser = False
  266. if newUser:
  267. imageId = create_image(cur, row[9][index], row[8])
  268. sql = "Insert into users (user_id, forum_id, name_user, status_user, reputation_user, interest_user, " \
  269. "signature_user, image_user, dateinserted_user) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
  270. recset = [userId, forumId,
  271. row[10][index],
  272. row[11][index] if row[11][index] != '-1' else None,
  273. row[12][index] if row[12][index] != '-1' else None,
  274. row[13][index] if row[13][index] != '-1' else None,
  275. row[14][index] if row[14][index] != '-1' else None,
  276. imageId,
  277. row[8]]
  278. cur.execute(sql, recset)
  279. else:
  280. # Tracking potential user changes
  281. sql = "select * from users where user_id = %(userId)s"
  282. cur.execute(sql, {'userId': userId})
  283. recset = cur.fetchall()
  284. imageId = recset[0]['image_user']
  285. if not is_same_image(cur, row[9][index], recset[0]['image_user']):
  286. imageId = create_image(cur, row[9][index], row[8])
  287. if (str(recset[0]['status_user']) != str(row[11][index] if row[11][index] != '-1' else None) or
  288. str(recset[0]['reputation_user']) != str(row[12][index] if row[12][index] != '-1' else None) or
  289. str(recset[0]['interest_user']) != str(row[13][index] if row[13][index] != '-1' else None) or
  290. str(recset[0]['signature_user']) != str(row[14][index] if row[14][index] != '-1' else None) or
  291. str(recset[0]['image_user']) != str(imageId)): # there was a change in the user information
  292. if (str(recset[0]['status_user']) != 'Nan' or
  293. str(recset[0]['reputation_user']) != 'Nan' or
  294. str(recset[0]['interest_user']) != 'Nan' or
  295. str(recset[0]['signature_user']) != 'Nan' or
  296. str(recset[0]['image_user']) != str(None)):
  297. userVersionId = int(getLastUserVersion(cur, userId) + 1)
  298. sql = "Insert into users_history (user_id, version_user, forum_id, name_user, status_user, reputation_user, interest_user, " \
  299. "signature_user, image_user, dateinserted_user) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
  300. recset = [userId, userVersionId, forumId,
  301. recset[0]['name_user'],
  302. recset[0]['status_user'],
  303. recset[0]['reputation_user'],
  304. recset[0]['interest_user'],
  305. recset[0]['signature_user'],
  306. recset[0]['image_user'],
  307. recset[0]['dateinserted_user']]
  308. cur.execute(sql, recset)
  309. sql = "Update users set status_user = %(status_user)s, reputation_user = %(reputation_user)s, " \
  310. "interest_user = %(interest_user)s, signature_user = %(signature_user)s, image_user = %(image_user)s, " \
  311. "dateinserted_user = %(dateinserted_user)s where user_id = %(userId)s"
  312. cur.execute(sql, {'status_user': row[11][index] if row[11][index] != '-1' else None,
  313. 'reputation_user': row[12][index] if row[12][index] != '-1' else None,
  314. 'interest_user': row[13][index] if row[13][index] != '-1' else None,
  315. 'signature_user': row[14][index] if row[14][index] != '-1' else None,
  316. 'image_user': imageId,
  317. 'dateinserted_user': row[8],
  318. 'userId': userId})
  319. return userId
  320. def create_posts(cur, row, forumId, topicId):
  321. if row[10] != "-1":
  322. for i in range(len(row[10])):
  323. userId = create_user(cur, row, forumId, i)
  324. postId = verifyPost(cur, topicId, userId, row[17][i])
  325. if not postId:
  326. postId = int(getLastPost(cur) + 1)
  327. newPost = True
  328. else:
  329. newPost = False
  330. if newPost:
  331. imageId = create_image(cur, row[18][i], row[8])
  332. sql = "Insert into posts (post_id, topic_id, user_id, content_post, feedback_post, image_post," \
  333. "dateadded_post, dateinserted_post) Values (%s, %s, %s, %s, %s, %s, %s, %s)"
  334. recset = [postId, topicId, userId,
  335. row[15][i] if row[15][i] != '-1' else None,
  336. row[16][i] if row[16][i] != '-1' else None,
  337. imageId,
  338. row[17][i],
  339. row[8]]
  340. cur.execute(sql, recset)
  341. else:
  342. # Tracking potential post changes
  343. sql = "select * from posts where post_id = %(postId)s"
  344. cur.execute(sql, {'postId': postId})
  345. recset = cur.fetchall()
  346. imageId = recset[0]['image_post']
  347. if not is_same_image(cur, row[18][i], recset[0]['image_post']):
  348. imageId = create_image(cur, row[18][i], row[8])
  349. if (str(recset[0]['content_post']) != str(row[15][i]) or
  350. str(recset[0]['feedback_post']) != str(row[16][i] if row[16][i] != '-1' else None) or
  351. str(recset[0]['image_post']) != str(imageId)): # there was a change in the post information
  352. postVersionId = int(getLastPostVersion(cur, postId) + 1)
  353. sql = "Insert into posts_history (post_id, version_post, topic_id, user_id, content_post, feedback_post, " \
  354. "image_post, dateadded_post, dateinserted_post) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s)"
  355. recset = [postId, postVersionId, topicId, userId,
  356. recset[0]['content_post'],
  357. recset[0]['feedback_post'],
  358. recset[0]['image_post'],
  359. recset[0]['dateadded_post'],
  360. recset[0]['dateinserted_post']]
  361. cur.execute(sql, recset)
  362. sql = "Update posts set content_post = %(content_post)s, feedback_post = %(feedback_post)s, " \
  363. "image_post = %(image_post)s, dateinserted_post = %(dateinserted_post)s where post_id = %(postId)s"
  364. cur.execute(sql, {'content_post': row[15][i] if row[15][i] != '-1' else None,
  365. 'feedback_post': row[16][i] if row[16][i] != '-1' else None,
  366. 'image_post': imageId,
  367. 'dateinserted_post': row[8],
  368. 'postId': postId})
  369. def is_same_image(cur, newBase64Image, oldImageId):
  370. if newBase64Image == "-1" and str(oldImageId) is str(None):
  371. return True
  372. if newBase64Image == "-1": # and str(oldImageId) != str(None)
  373. return False
  374. if str(oldImageId) == str(None): # and newBase64Image != "-1"
  375. return False
  376. sql = "select * from forum_images where image_id = %(imageId)s"
  377. cur.execute(sql, {'imageId': oldImageId})
  378. recset = cur.fetchall()
  379. decImage = decode_decrypt_image_in_base64(newBase64Image)
  380. hash1 = generate_image_hash(decImage)
  381. hash2 = recset[0]['hash_image']
  382. if hash1 == hash2:
  383. return True
  384. features1 = json.loads(extract_hidden_layer_output(decImage))
  385. features2 = json.loads(recset[0]['resnet50_image'])
  386. similarity = 1 - distance.cosine(features1, features2)
  387. return similarity >= 0.8
  388. def create_image(cur, base64Image, date):
  389. if base64Image == "-1":
  390. return None
  391. decImage = decode_decrypt_image_in_base64(base64Image)
  392. imageId = verifyImage(cur, decImage)
  393. if not imageId:
  394. imageId = int(getLastImage(cur) + 1)
  395. newImage = True
  396. else:
  397. newImage = False
  398. if newImage:
  399. hashImage = generate_image_hash(decImage)
  400. hloFeatures = extract_hidden_layer_output(decImage)
  401. keypoints, descriptors = extract_keypoints(decImage)
  402. sql = "Insert into forum_images(image_id, hash_image, base64_image, " \
  403. "resnet50_image, siftkey_image, siftdesc_image, dateinserted_image) " \
  404. "Values (%s, %s, %s, %s, %s, %s, %s)"
  405. recset = [imageId,
  406. hashImage,
  407. base64Image,
  408. hloFeatures,
  409. keypoints,
  410. descriptors,
  411. date]
  412. cur.execute(sql, recset)
  413. return imageId
  414. def create_status(cur, forumId, date, listings, descriptions, status):
  415. date = datetime.strptime(date, "%m%d%Y")
  416. # getting last Fridays a reference date
  417. date_reference = date + relativedelta(weekday=FR(-1))
  418. # checking if status already exists
  419. sql = "select * from forums_status where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
  420. cur.execute(sql, {'forum_id': forumId, 'date_inserted': date})
  421. recset = cur.fetchall()
  422. if recset:
  423. sql = "Update forums_status set listings = %(listings)s, descriptions = %(descriptions)s, status = %(status)s, date_reference = %(date_reference)s " \
  424. "where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
  425. recset = {'listings': listings, 'descriptions': descriptions, 'status': status, 'date_reference': date_reference, 'forum_id': forumId, 'date_inserted': date}
  426. else:
  427. sql = "Insert into forums_status (forum_id, date_inserted, listings, descriptions, status, date_reference) Values (%s, %s, %s, %s, %s, %s)"
  428. recset = [forumId, date, listings, descriptions, status, date_reference]
  429. cur.execute(sql, recset)
  430. def create_database(cur, con):
  431. try:
  432. sql = "create table forum_images(image_id integer not null, hash_image character varying(64) not null, base64_image text not null, " \
  433. "resnet50_image text null, siftkey_image text null, siftdesc_image text null, " \
  434. "dateinserted_image timestamp(6) with time zone not null, " \
  435. "constraint forum_images_pk primary key (image_id))"
  436. cur.execute(sql)
  437. sql = "create unique index unique_forum_image ON forum_images USING btree (hash_image ASC NULLS LAST)"
  438. cur.execute(sql)
  439. sql = "create table forums (forum_id integer NOT NULL, name_forum character varying(255) NOT NULL, url_forum " \
  440. "character varying(255) NOT null, dateinserted_forum timestamp(6) with time zone NOT NULL, " \
  441. "constraint forums_pk primary key (forum_id))"
  442. cur.execute(sql)
  443. sql = "create unique index unique_forum ON forums USING btree (name_forum ASC NULLS LAST)"
  444. cur.execute(sql)
  445. sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, " \
  446. "listings integer NOT NULL, descriptions integer NOT NULL, status bit(1) NOT NULL, date_reference date NOT NULL, " \
  447. "constraint forums_status_pk PRIMARY KEY (forum_id, date_inserted), " \
  448. "constraint forums_status_fk FOREIGN KEY (forum_id) REFERENCES forums (forum_id))"
  449. cur.execute(sql)
  450. sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \
  451. "255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \
  452. "interest_user character varying(10000) null, signature_user character varying(10000) null, " \
  453. "image_user integer null, dateinserted_user timestamp(6) with time zone NOT NULL, " \
  454. "constraint users_pk primary key (user_id), " \
  455. "constraint users_forum_id_fk foreign key (forum_id) references forums (forum_id), " \
  456. "constraint users_image_id_fk foreign key (image_user) references forum_images (image_id))"
  457. cur.execute(sql)
  458. sql = "create unique index unique_user ON users USING btree (forum_id ASC NULLS LAST, name_user ASC NULLS LAST)"
  459. cur.execute(sql)
  460. sql = "create table users_history(user_id integer NOT NULL, version_user integer not null, forum_id integer NOT NULL, " \
  461. "name_user character varying(255) NOT NULL, status_user character varying(255) null, " \
  462. "reputation_user character varying(255) null, interest_user character varying(10000) null, " \
  463. "signature_user character varying(10000) null, image_user integer null, " \
  464. "dateinserted_user timestamp(6) with time zone NOT NULL, " \
  465. "constraint users_history_pk primary key (user_id, version_user), " \
  466. "constraint users_history_user_id_fk foreign key (user_id) references users (user_id), " \
  467. "constraint users_history_forum_id_fk foreign key (forum_id) references forums (forum_id), " \
  468. "constraint users_history_image_id_fk foreign key (image_user) references forum_images (image_id))"
  469. cur.execute(sql)
  470. sql = "create table topics(topic_id integer NOT NULL, forum_id integer NOT NULL, author_id integer NOT NULL, " \
  471. "title_topic character varying(255) NOT NULL, board_topic character varying(255) NOT NULL, views_topic integer null, " \
  472. "posts_topic integer null, href_topic character varying(255) NOT null, dateadded_topic timestamp(6) with time zone null, " \
  473. "dateinserted_topic timestamp(6) with time zone NOT NULL, classification_topic double precision null, " \
  474. "constraint topics_pk primary key (topic_id), " \
  475. "constraint topics_author_id_fk foreign key (author_id) references users (user_id), " \
  476. "constraint topics_forum_id_fk foreign key (forum_id) references forums (forum_id))"
  477. cur.execute(sql)
  478. sql = "create unique index unique_topic ON topics USING btree (forum_id ASC NULLS LAST, href_topic ASC NULLS LAST)"
  479. cur.execute(sql)
  480. sql = "create table topics_history(topic_id integer NOT NULL, version_topic integer not null, forum_id integer NOT NULL, " \
  481. "author_id integer NOT NULL, title_topic character varying(255) NOT NULL, board_topic character varying(255) NOT NULL, " \
  482. "views_topic integer null, posts_topic integer null, href_topic character varying(255) NOT null, " \
  483. "dateadded_topic timestamp(6) with time zone null, dateinserted_topic timestamp(6) with time zone NOT NULL, " \
  484. "classification_topic double precision null, " \
  485. "constraint topics_history_pk primary key (topic_id, version_topic), " \
  486. "constraint topics_history_topic_id_fk foreign key (topic_id) references topics (topic_id), " \
  487. "constraint topics_history_author_id_f foreign key (author_id) references users (user_id), " \
  488. "constraint topics_history_board_id_fk foreign key (forum_id) references forums (forum_id))"
  489. cur.execute(sql)
  490. sql = "create table posts(post_id integer NOT NULL, topic_id integer NOT NULL, " \
  491. "user_id integer NOT NULL, content_post character varying(1000000) NOT null, feedback_post integer null, " \
  492. "image_post integer null, dateadded_post timestamp(6) with time zone NOT NULL, " \
  493. "dateinserted_post timestamp(6) with time zone NOT NULL, " \
  494. "constraint posts_pk primary key (post_id), " \
  495. "constraint posts_user_id_fk foreign key (user_id) references users (user_id), " \
  496. "constraint posts_topic_id_fk foreign key (topic_id) references topics (topic_id), " \
  497. "constraint posts_image_id_fk foreign key (image_post) references forum_images (image_id))"
  498. cur.execute(sql)
  499. sql = "create unique index unique_post ON posts USING btree (topic_id ASC NULLS LAST, user_id ASC NULLS LAST, " \
  500. "dateadded_post ASC NULLS LAST)"
  501. cur.execute(sql)
  502. sql = "create table posts_history(post_id integer NOT NULL, version_post integer not null, topic_id integer NOT NULL, " \
  503. "user_id integer NOT NULL, content_post character varying(1000000) NOT null, feedback_post integer null, " \
  504. "image_post integer null, dateadded_post timestamp(6) with time zone NOT NULL, " \
  505. "dateinserted_post timestamp(6) with time zone NOT NULL, " \
  506. "constraint posts_history_pk primary key (post_id, version_post), " \
  507. "constraint posts_history_user_id_fk foreign key (user_id) references users (user_id), " \
  508. "constraint posts_history_topic_id_fk foreign key (topic_id) references topics (topic_id), " \
  509. "constraint posts_history_post_id_fk foreign key (post_id) references posts (post_id), " \
  510. "constraint posts_history_image_id_fk foreign key (image_post) references forum_images (image_id))"
  511. cur.execute(sql)
  512. con.commit()
  513. except:
  514. con.rollback()
  515. trace = traceback.format_exc()
  516. if (trace.find("already exists")==-1):
  517. print ("There was a problem during the database creation." )
  518. traceback.print_exc()
  519. raise SystemExit