数据库连接方式改为共用连接，降低时间开销
This commit is contained in:
xaoyaoo
parent
0a6226c6a0
commit
db50384808
@ -6,10 +6,10 @@
|
||||
# Date: 2023/10/14
|
||||
# -------------------------------------------------------------------------------
|
||||
from .wx_info import BiasAddr, read_info, get_wechat_db, batch_decrypt, decrypt, get_core_db
|
||||
from .wx_info import merge_copy_db, merge_msg_db, merge_media_msg_db, merge_db, decrypt_merge,merge_real_time_db
|
||||
from .wx_info import merge_copy_db, merge_msg_db, merge_media_msg_db, merge_db, decrypt_merge, merge_real_time_db
|
||||
from .analyzer.db_parsing import read_img_dat, read_emoji, decompress_CompressContent, read_audio_buf, read_audio, \
|
||||
parse_xml_string, read_BytesExtra
|
||||
from .analyzer import export_csv,export_json
|
||||
from .analyzer import export_csv, export_json, DBPool
|
||||
from .ui import app_show_chat, get_user_list, export
|
||||
from .server import start_falsk
|
||||
|
||||
@ -26,3 +26,5 @@ except:
|
||||
PYWXDUMP_ROOT_PATH = os.path.dirname(__file__)
|
||||
|
||||
__version__ = "2.4.60"
|
||||
|
||||
db_init = DBPool("DBPOOL_INIT")
|
||||
@ -9,4 +9,4 @@ from .db_parsing import read_img_dat, read_emoji, decompress_CompressContent, re
|
||||
parse_xml_string, read_BytesExtra
|
||||
from .export_chat import export_csv, get_contact_list, get_chatroom_list, get_msg_list, get_chat_count, export_json, \
|
||||
get_all_chat_count
|
||||
from .utils import get_type_name, get_name_typeid
|
||||
from .utils import get_type_name, get_name_typeid,DBPool
|
||||
|
||||
@ -20,7 +20,7 @@ import json
|
||||
import time
|
||||
from functools import wraps
|
||||
|
||||
from .utils import get_md5, attach_databases, execute_sql, get_type_name, match_BytesExtra
|
||||
from .utils import get_md5, attach_databases, execute_sql, get_type_name, match_BytesExtra, DBPool
|
||||
from .db_parsing import parse_xml_string, decompress_CompressContent, read_BytesExtra
|
||||
|
||||
|
||||
@ -31,17 +31,13 @@ def get_contact(MicroMsg_db_path, wx_id):
|
||||
:param wx_id: 微信id
|
||||
:return: 联系人信息
|
||||
"""
|
||||
db = sqlite3.connect(MicroMsg_db_path)
|
||||
cursor = db.cursor()
|
||||
with DBPool(MicroMsg_db_path) as db:
|
||||
# 获取username是wx_id的用户
|
||||
sql = ("SELECT A.UserName, A.NickName, A.Remark,A.Alias,A.Reserved6,B.bigHeadImgUrl "
|
||||
"FROM Contact A,ContactHeadImgUrl B "
|
||||
f"WHERE A.UserName = '{wx_id}' AND A.UserName = B.usrName "
|
||||
"ORDER BY NickName ASC;")
|
||||
cursor.execute(sql)
|
||||
result = cursor.fetchone()
|
||||
cursor.close()
|
||||
db.close()
|
||||
result = execute_sql(db, sql)
|
||||
print('联系人信息：', result)
|
||||
if not result:
|
||||
print('居然没找到！')
|
||||
@ -59,23 +55,18 @@ def get_contact_list(MicroMsg_db_path):
|
||||
"""
|
||||
users = []
|
||||
# 连接 MicroMsg.db 数据库，并执行查询
|
||||
db = sqlite3.connect(MicroMsg_db_path)
|
||||
cursor = db.cursor()
|
||||
with DBPool(MicroMsg_db_path) as db:
|
||||
sql = ("SELECT A.UserName, A.NickName, A.Remark,A.Alias,A.Reserved6,B.bigHeadImgUrl "
|
||||
"FROM Contact A,ContactHeadImgUrl B "
|
||||
"where UserName==usrName "
|
||||
"ORDER BY NickName ASC;")
|
||||
cursor.execute(sql)
|
||||
result = cursor.fetchall()
|
||||
|
||||
result = execute_sql(db, sql)
|
||||
for row in result:
|
||||
# 获取用户名、昵称、备注和聊天记录数量
|
||||
username, nickname, remark, Alias, describe, headImgUrl = row
|
||||
users.append(
|
||||
{"username": username, "nickname": nickname, "remark": remark, "account": Alias, "describe": describe,
|
||||
"headImgUrl": headImgUrl})
|
||||
cursor.close()
|
||||
db.close()
|
||||
return users
|
||||
|
||||
|
||||
@ -87,15 +78,12 @@ def get_chatroom_list(MicroMsg_db_path):
|
||||
"""
|
||||
rooms = []
|
||||
# 连接 MicroMsg.db 数据库，并执行查询
|
||||
db = sqlite3.connect(MicroMsg_db_path)
|
||||
|
||||
with DBPool(MicroMsg_db_path) as db:
|
||||
sql = ("SELECT A.ChatRoomName,A.UserNameList, A.DisplayNameList, B.Announcement,B.AnnouncementEditor "
|
||||
"FROM ChatRoom A,ChatRoomInfo B "
|
||||
"where A.ChatRoomName==B.ChatRoomName "
|
||||
"ORDER BY A.ChatRoomName ASC;")
|
||||
|
||||
result = execute_sql(db, sql)
|
||||
db.close()
|
||||
for row in result:
|
||||
# 获取用户名、昵称、备注和聊天记录数量
|
||||
ChatRoomName, UserNameList, DisplayNameList, Announcement, AnnouncementEditor = row
|
||||
@ -116,18 +104,13 @@ def get_room_user_list(MSG_db_path, selected_talker):
|
||||
"""
|
||||
|
||||
# 连接 MSG_ALL.db 数据库，并执行查询
|
||||
db1 = sqlite3.connect(MSG_db_path)
|
||||
cursor1 = db1.cursor()
|
||||
|
||||
with DBPool(MSG_db_path) as db1:
|
||||
sql = (
|
||||
"SELECT localId, IsSender, StrContent, StrTalker, Sequence, Type, SubType,CreateTime,MsgSvrID,DisplayContent,CompressContent,BytesExtra,ROW_NUMBER() OVER (ORDER BY CreateTime ASC) AS id "
|
||||
"FROM MSG WHERE StrTalker=? "
|
||||
"ORDER BY CreateTime ASC")
|
||||
|
||||
cursor1.execute(sql, (selected_talker,))
|
||||
result1 = cursor1.fetchall()
|
||||
cursor1.close()
|
||||
db1.close()
|
||||
result1 = execute_sql(db1, sql, (selected_talker,))
|
||||
user_list = []
|
||||
read_user_wx_id = []
|
||||
for row in result1:
|
||||
@ -159,22 +142,18 @@ def get_msg_list(MSG_db_path, selected_talker="", start_index=0, page_size=500):
|
||||
"""
|
||||
|
||||
# 连接 MSG_ALL.db 数据库，并执行查询
|
||||
db1 = sqlite3.connect(MSG_db_path)
|
||||
cursor1 = db1.cursor()
|
||||
with DBPool(MSG_db_path) as db1:
|
||||
if selected_talker:
|
||||
sql = (
|
||||
"SELECT localId, IsSender, StrContent, StrTalker, Sequence, Type, SubType,CreateTime,MsgSvrID,DisplayContent,CompressContent,BytesExtra,ROW_NUMBER() OVER (ORDER BY CreateTime ASC) AS id "
|
||||
"FROM MSG WHERE StrTalker=? "
|
||||
"ORDER BY CreateTime ASC LIMIT ?,?")
|
||||
cursor1.execute(sql, (selected_talker, start_index, page_size))
|
||||
result1 = execute_sql(db1,sql, (selected_talker, start_index, page_size))
|
||||
else:
|
||||
sql = (
|
||||
"SELECT localId, IsSender, StrContent, StrTalker, Sequence, Type, SubType,CreateTime,MsgSvrID,DisplayContent,CompressContent,BytesExtra,ROW_NUMBER() OVER (ORDER BY CreateTime ASC) AS id "
|
||||
"FROM MSG ORDER BY CreateTime ASC LIMIT ?,?")
|
||||
cursor1.execute(sql, (start_index, page_size))
|
||||
result1 = cursor1.fetchall()
|
||||
cursor1.close()
|
||||
db1.close()
|
||||
result1 = execute_sql(db1,sql, (start_index, page_size))
|
||||
|
||||
data = []
|
||||
for row in result1:
|
||||
@ -301,14 +280,13 @@ def get_chat_count(MSG_db_path: [str, list], username: str = ""):
|
||||
sql = f"SELECT StrTalker,COUNT(*) FROM MSG WHERE StrTalker='{username}';"
|
||||
else:
|
||||
sql = f"SELECT StrTalker, COUNT(*) FROM MSG GROUP BY StrTalker ORDER BY COUNT(*) DESC;"
|
||||
db1 = sqlite3.connect(MSG_db_path)
|
||||
result = execute_sql(db1, sql)
|
||||
|
||||
with DBPool(MSG_db_path) as db1:
|
||||
result = execute_sql(db1, sql)
|
||||
chat_counts = {}
|
||||
for row in result:
|
||||
username, chat_count = row
|
||||
chat_counts[username] = chat_count
|
||||
db1.close()
|
||||
return chat_counts
|
||||
|
||||
|
||||
@ -319,16 +297,15 @@ def get_all_chat_count(MSG_db_path: [str, list]):
|
||||
:return: 聊天记录数量
|
||||
"""
|
||||
sql = f"SELECT COUNT(*) FROM MSG;"
|
||||
db1 = sqlite3.connect(MSG_db_path)
|
||||
with DBPool(MSG_db_path) as db1:
|
||||
result = execute_sql(db1, sql)
|
||||
if result and len(result) > 0:
|
||||
chat_counts = result[0][0]
|
||||
db1.close()
|
||||
return chat_counts
|
||||
db1.close()
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
def export_csv(username, outpath, MSG_ALL_db_path, page_size=5000):
|
||||
if not os.path.exists(outpath):
|
||||
outpath = os.path.join(os.getcwd(), "export" + os.sep + username)
|
||||
|
||||
@ -6,7 +6,9 @@
|
||||
# Date: 2023/12/03
|
||||
# -------------------------------------------------------------------------------
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
|
||||
def read_dict_all_values(data):
|
||||
@ -115,6 +117,12 @@ def get_name_typeid(type_name: str):
|
||||
(43, 0): "视频",
|
||||
(47, 0): "动画表情",
|
||||
|
||||
(37, 0): "添加好友", # 感谢 https://github.com/zhyc9de
|
||||
(42, 0): "推荐公众号", # 感谢 https://github.com/zhyc9de
|
||||
(48, 0): "地图信息", # 感谢 https://github.com/zhyc9de
|
||||
(49, 40): "分享收藏夹", # 感谢 https://github.com/zhyc9de
|
||||
(49, 53): "接龙", # 感谢 https://github.com/zhyc9de
|
||||
|
||||
(49, 0): "文件",
|
||||
(49, 1): "类似文字消息而不一样的消息",
|
||||
(49, 5): "卡片式链接",
|
||||
@ -153,6 +161,57 @@ def get_md5(data):
|
||||
return md5.hexdigest()
|
||||
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
def get_thread_id():
|
||||
current_thread = threading.current_thread()
|
||||
thread_id = current_thread.ident
|
||||
return thread_id
|
||||
|
||||
|
||||
class DBPool:
|
||||
__db_pool = {}
|
||||
__thread_pool = {}
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
if not hasattr(cls, '_instance'):
|
||||
cls._instance = super(DBPool, cls).__new__(cls)
|
||||
return cls._instance
|
||||
|
||||
@classmethod
|
||||
def create_connection(cls, db_path):
|
||||
if db_path == "DBPOOL_INIT":
|
||||
return
|
||||
if not os.path.exists(db_path):
|
||||
raise FileNotFoundError(f"数据库文件不存在：{db_path}")
|
||||
|
||||
if db_path not in cls.__db_pool:
|
||||
cls.__db_pool[db_path] = sqlite3.connect(db_path, check_same_thread=False)
|
||||
print(f"数据库连接成功")
|
||||
print(f"数据库连接成功 1")
|
||||
print(cls.__db_pool)
|
||||
cls.connection = cls.__db_pool[db_path]
|
||||
|
||||
def __init__(self, db_path):
|
||||
if db_path == "DBPOOL_INIT":
|
||||
return
|
||||
self.db_path = db_path
|
||||
if db_path not in self.__db_pool:
|
||||
self.create_connection(db_path)
|
||||
self.connection = self.__db_pool.get(db_path)
|
||||
|
||||
def __enter__(self):
|
||||
return self.connection
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.connection = None
|
||||
|
||||
def close(self):
|
||||
self.connection.close()
|
||||
self.connection = None
|
||||
|
||||
|
||||
def attach_databases(connection, databases):
|
||||
"""
|
||||
将多个数据库附加到给定的SQLite连接。
|
||||
@ -198,7 +257,7 @@ def execute_sql(connection, sql, params=None):
|
||||
else:
|
||||
cursor.execute(sql)
|
||||
return cursor.fetchall()
|
||||
except Exception as e:
|
||||
except Exception as e1:
|
||||
try:
|
||||
connection.text_factory = bytes
|
||||
cursor = connection.cursor()
|
||||
@ -209,8 +268,8 @@ def execute_sql(connection, sql, params=None):
|
||||
rdata = cursor.fetchall()
|
||||
connection.text_factory = str
|
||||
return rdata
|
||||
except Exception as e:
|
||||
print(f"**********\nSQL: {sql}\nparams: {params}\n{e}\n**********")
|
||||
except Exception as e2:
|
||||
print(f"**********\nSQL: {sql}\nparams: {params}\n{e1}\n{e2}\n**********")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user