diff --git a/pyminer/features/preprocess/preprocess.py b/pyminer/features/preprocess/preprocess.py index b126c9b44f01fcb9614b2f1351f8ae8f7cf28575..d84403628a6119976c8e5d3988fa0df25fa6b8a7 100644 --- a/pyminer/features/preprocess/preprocess.py +++ b/pyminer/features/preprocess/preprocess.py @@ -12,6 +12,9 @@ from pandas.api.types import is_string_dtype # 导入PyQt5模块 from PyQt5.Qt import * +# 导入基础类模块 +from pyminer.ui.base.widgets.generalwidgets.window import DataProcessDialog + # 导入数据相关操作模块 from pyminer.ui.data.data_filter import Ui_Form as DataFilter_Ui_Form # 数据筛选 from pyminer.ui.data.data_role import Ui_Form as DataRole_Ui_Form # 数据角色 @@ -33,12 +36,13 @@ from pyminer.ui.data.data_column_encode import Ui_Form as DataColumnEncode_Ui_Fo from pyminer.ui.data.data_column_name import Ui_Form as DataColumnName_Ui_Form from pyminer.ui.data.data_repace import Ui_Form as DataReplace_Ui_Form from pyminer.share.exceptionhandler import exception_handler + # 定义日志输出格式 logging.basicConfig(format="%(asctime)s %(name)s:%(levelname)s:%(message)s", datefmt="%Y-%m-%d %H:%M:%S", level=logging.INFO) -class DataInfoForm(QDialog, DataInfo_Ui_Form): +class DataInfoForm(DataProcessDialog, DataInfo_Ui_Form): """ 打开"数据-数据信息" """ @@ -47,7 +51,7 @@ class DataInfoForm(QDialog, DataInfo_Ui_Form): super().__init__(parent) self.setupUi(self) self.center() - self.all_dataset = dict() + self.data_host = None self.all_dataset_name = list() self.current_dataset_name = '' # 当前数据集名称 @@ -64,27 +68,6 @@ class DataInfoForm(QDialog, DataInfo_Ui_Form): self.lineEdit_dataset_name.textChanged.connect(self.change_dataset_info) # ================================自定义槽函数========================= - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def info_init(self): if any(self.info): @@ -118,21 +101,25 @@ class DataInfoForm(QDialog, DataInfo_Ui_Form): def change_dataset_info(self): # 修改当前数据集的数据信息 - self.lineEdit_path.setText(self.all_dataset.get(self.current_dataset_name + ".path")) - self.lineEdit_row.setText(self.all_dataset.get(self.current_dataset_name + ".row")) - self.lineEdit_col.setText(self.all_dataset.get(self.current_dataset_name + ".col")) - self.lineEdit_file_size.setText(self.all_dataset.get(self.current_dataset_name + ".file_size")) - self.lineEdit_memory_usage.setText( - self.all_dataset.get(self.current_dataset_name + ".memory_usage")) - self.lineEdit_create_time.setText( - self.all_dataset.get(self.current_dataset_name + ".create_time")) - self.lineEdit_update_time.setText( - self.all_dataset.get(self.current_dataset_name + ".update_time")) - self.info = self.all_dataset.get(self.current_dataset_name + ".info") - self.info_init() - - -class DataFilterForm(QWidget, DataFilter_Ui_Form): + if self.data_host is not None: + info_dic = self.data_host.get_info(self.current_dataset_name) + + self.lineEdit_path.setText(info_dic['path']) # self.data_manager.get(self.current_dataset_name + ".path")) + self.lineEdit_row.setText(info_dic['row']) # self.data_manager.get(self.current_dataset_name + ".row")) + self.lineEdit_col.setText(info_dic['col']) # self.data_manager.get(self.current_dataset_name + ".col")) + self.lineEdit_file_size.setText( + info_dic['file_size']) # self.data_manager.get(self.current_dataset_name + ".file_size")) + self.lineEdit_memory_usage.setText(info_dic['memory_usage']) # + # self.data_manager.get(self.current_dataset_name + ".memory_usage")) + self.lineEdit_create_time.setText(info_dic['create_time']) + # self.data_manager.get(self.current_dataset_name + ".create_time")) + self.lineEdit_update_time.setText(info_dic['update_time']) + # self.data_manager.get(self.current_dataset_name + ".update_time")) + self.info = info_dic['info'] # self.data_manager.get(self.current_dataset_name + ".info") + self.info_init() + + +class DataFilterForm(DataProcessDialog, DataFilter_Ui_Form): """ 打开"数据-筛选"窗口 """ @@ -142,21 +129,8 @@ class DataFilterForm(QWidget, DataFilter_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - -class DataRoleForm(QDialog, DataRole_Ui_Form): +class DataRoleForm(DataProcessDialog, DataRole_Ui_Form): """ "新建窗口" """ @@ -169,7 +143,7 @@ class DataRoleForm(QDialog, DataRole_Ui_Form): self.current_dataset = pd.DataFrame() # 当前数据集 self.current_dataset_name = "" - self.all_dataset = dict() + self.data_manager = dict() self.filter_dataset = pd.DataFrame() # 预览筛选后内容 self.role_dataset = pd.DataFrame() # 预览筛选后内容 @@ -181,28 +155,7 @@ class DataRoleForm(QDialog, DataRole_Ui_Form): self.lineEdit_col_find.textChanged.connect(self.change_find) self.comboBox_columns.currentTextChanged.connect(self.change_column) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") + # ================================自定义槽函数========================= def dataset_export(self): fileName_choose, filetype = QFileDialog.getSaveFileName(self, @@ -302,20 +255,20 @@ class DataRoleForm(QDialog, DataRole_Ui_Form): self.tableWidget_dataset.setItem(i, j, newItem) -class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): +class DataRowFilterForm(DataProcessDialog, DataRowFilter_Ui_Form): """ - 打开"数据-行筛选"窗口 + 打开"数据-行筛选"窗口,也就是‘筛选。’ """ signal_data_change = pyqtSignal(str, dict, str, str, str, str, str) # 自定义信号,用于修改数据 - def __init__(self, parent=None): + def __init__(self, parent=None, data_manager=None): super().__init__(parent) self.setupUi(self) self.center() self.current_dataset = pd.DataFrame() self.current_dataset_name = '' - self.all_dataset = {} + self.data_manager = data_manager self.current_dataset_columns = [] self.data_type = [] self.filter_dataset = pd.DataFrame() @@ -342,28 +295,7 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): self.lineEdit_col_find.textChanged.connect(self.exec_filter) self.comboBox_dtype.currentIndexChanged.connect(self.exec_filter) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def dataset_init(self): self.filter_dataset = self.current_dataset.copy().head(100) @@ -454,6 +386,7 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): else: self.comboBox_col_condition.clear() self.comboBox_col_condition.addItems(['模糊匹配', 'in', 'not in']) + @exception_handler def filter_column(self): # 根据列筛选 @@ -473,7 +406,7 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): elif self.comboBox_col_condition.currentText() == "<=": self.filter_dataset = data[data[col] <= float(content)] else: - content=content.lower() + content = content.lower() if self.comboBox_col_condition.currentText() == "模糊匹配": self.filter_dataset = data[data[col].map(str.lower).str.contains(content)] elif self.comboBox_col_condition.currentText() == "in": @@ -485,7 +418,7 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): def filter_dtype(self): data = self.current_dataset.copy() dtype = self.comboBox_dtype.currentText() # 当前要筛选的数据类型 - if dtype =="全部": + if dtype == "全部": self.data_preview(data) return self.filter_dataset = data.select_dtypes(include=dtype) @@ -514,10 +447,11 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): if ok and (len(dataset_name) != 0): logging.info("发射导入数据信号") if len(self.filter_dataset) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + property_dic = self.data_manager.get_info(self.current_dataset_name) + create_time = property_dic['create_time'] update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = property_dic['path'] + file_size = property_dic['file_size'] remarks = '' self.signal_data_change.emit(dataset_name, self.filter_dataset.to_dict(), @@ -537,10 +471,11 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): if reply == QMessageBox.Yes: logging.info("发射导入数据信号") if len(self.filter_dataset) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + info_dic = self.data_manager.get_info(self.current_dataset_name) + create_time = info_dic['create_time'] update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = info_dic['path'] + file_size = info_dic['file_size'] remarks = '' self.signal_data_change.emit(self.current_dataset_name, self.filter_dataset.to_dict(), @@ -555,7 +490,7 @@ class DataRowFilterForm(QDialog, DataRowFilter_Ui_Form): self.close() -class DataColumnDescForm(QDialog, Columns_desc_Ui_Form): +class DataColumnDescForm(DataProcessDialog, Columns_desc_Ui_Form): """ 打开"数据-列"窗口 """ @@ -588,28 +523,7 @@ class DataColumnDescForm(QDialog, Columns_desc_Ui_Form): self.pushButton_group_down.clicked.connect(self.var_group_down) self.pushButton_group_del.clicked.connect(self.var_group_del) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def slot_var_change(self): # 只允许查看一个变量的描述 @@ -778,7 +692,7 @@ class DataColumnDescForm(QDialog, Columns_desc_Ui_Form): self.tableWidget_dataset.setItem(i, j, newItem) -class DataDeleteRowForm(QWidget, DataDeleteRow_Ui_Form): +class DataDeleteRowForm(DataProcessDialog, DataDeleteRow_Ui_Form): """ 打开"数据-删除行"窗口 """ @@ -788,21 +702,8 @@ class DataDeleteRowForm(QWidget, DataDeleteRow_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - -class DataDeleteColumnForm(QWidget, DataDeleteColumn_Ui_Form): +class DataDeleteColumnForm(DataProcessDialog, DataDeleteColumn_Ui_Form): """ 打开"数据-删除列"窗口 """ @@ -812,21 +713,8 @@ class DataDeleteColumnForm(QWidget, DataDeleteColumn_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - -class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): +class DataMergeVerticalForm(DataProcessDialog, DataMergeVertical_Ui_Form): """ 打开"数据-纵向合并"窗口 """ @@ -838,7 +726,7 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): self.center() self.current_dataset_name = "" - self.all_dataset = {} # 定义“全部数据集”为一个字典 + self.data_manager = None # 定义“全部数据集”为一个字典 self.all_dataset_name = () self.dataset_alter = pd.DataFrame() # 修改后的数据 @@ -866,29 +754,7 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): self.pushButton_start_down.clicked.connect(self.dataset_start_down) self.pushButton_append_down.clicked.connect(self.dataset_append_down) - # ================================事件处理函数========================= - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def slot_listWidget_start_change(self): self.current_dataset_name = self.listWidget_start.item(0).text() @@ -1000,12 +866,12 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): def dataset_merge_vertical(self): dataset_name = self.listWidget_start.item(0).text() - dataset_start = self.all_dataset.get(dataset_name) + dataset_start = self.data_manager.get(dataset_name) dataset_merge_list_name = [] dataset_merge_list = [dataset_start, ] for i in range(self.listWidget_append.count()): dataset_merge_list_name.append(self.listWidget_append.item(i).text()) - dataset_merge_list.append(self.all_dataset.get(self.listWidget_append.item(i).text())) + dataset_merge_list.append(self.data_manager.get(self.listWidget_append.item(i).text())) if self.listWidget_start.count() < 1: QMessageBox.information(self, "注意", "请选择起始数据集", QMessageBox.Yes | QMessageBox.No, QMessageBox.Yes) @@ -1020,10 +886,10 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): if reply == QMessageBox.Yes: logging.info("发射导入数据信号") if len(self.dataset_alter) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(self.current_dataset_name, self.dataset_alter.to_dict(), @@ -1044,10 +910,10 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): if ok and (len(dataset_name) != 0): logging.info("发射导入数据信号") if len(self.dataset_alter) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(dataset_name, self.dataset_alter.to_dict(), @@ -1062,7 +928,7 @@ class DataMergeVerticalForm(QDialog, DataMergeVertical_Ui_Form): self.close() -class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): +class DataMergeHorizontalForm(DataProcessDialog, DataMergeHorizontal_Ui_Form): """ 打开"数据-横向合并"窗口 """ @@ -1074,7 +940,7 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): self.center() self.current_dataset_name = "" - self.all_dataset = {} # 定义“全部数据集”为一个字典 + self.data_manager = {} # 定义“全部数据集”为一个字典 self.dataset_alter = pd.DataFrame() self.listWidget_dataset.setSelectionMode(QAbstractItemView.ExtendedSelection) # 设置为按住ctrl可以多选 @@ -1095,29 +961,7 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): self.pushButton_append_down.clicked.connect(self.dataset_append_down) self.pushButton_append_del.clicked.connect(self.dataset_append_del) - # ================================事件处理函数========================= - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def slot_listWidget_start_change(self): self.current_dataset_name = self.listWidget_start.item(0).text() @@ -1229,14 +1073,14 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): def dataset_merge_horizontal(self): dataset_name = self.listWidget_start.item(0).text() # dataset_name - dataset_start = self.all_dataset.get(dataset_name) # dataset_start + dataset_start = self.data_manager.get(dataset_name) # dataset_start dataset_merge_list_name = [] # 要合并的数据集名称 dataset_merge_list = [] # 要合并的数据 for i in range(self.listWidget_append.count()): # 遍历列表n-1次 if i < self.listWidget_append.count(): dataset_name = self.listWidget_append.item(i).text() dataset_merge_list_name.append(dataset_name) - dataset = self.all_dataset.get(self.listWidget_append.item(i).text()).copy() # 避免修改原始数据 + dataset = self.data_manager.get(self.listWidget_append.item(i).text()).copy() # 避免修改原始数据 dataset.columns = [x + "_" + dataset_name.split('.')[0] for x in dataset.columns] # 为列名添加数据集后缀 dataset_merge_list.append(dataset) @@ -1256,10 +1100,10 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): if reply == QMessageBox.Yes: logging.info("发射导入数据信号") if len(self.dataset_alter) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(self.current_dataset_name, self.dataset_alter.to_dict(), @@ -1280,10 +1124,10 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): if ok and (len(dataset_name) != 0): logging.info("发射导入数据信号") if len(self.dataset_alter) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(dataset_name, self.dataset_alter.to_dict(), @@ -1298,7 +1142,7 @@ class DataMergeHorizontalForm(QDialog, DataMergeHorizontal_Ui_Form): self.close() -class DataPartitionForm(QDialog, DataPartition_Ui_Form): +class DataPartitionForm(DataProcessDialog, DataPartition_Ui_Form): """ 打开"数据-数据分区"窗口 """ @@ -1309,51 +1153,20 @@ class DataPartitionForm(QDialog, DataPartition_Ui_Form): self.setupUi(self) self.center() - self.current_dataset = pd.DataFrame() self.current_dataset_name = '' - self.widget_part_2.hide() self.widget_part_3.hide() self.widget_part_4.hide() self.widget_part_other.setVisible(False) - - - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") - - - - - - -class DataNewColumnForm(QWidget, DataNewColumn_Ui_Form): +class DataNewColumnForm(DataProcessDialog, DataNewColumn_Ui_Form): """ - 打开"从sas导入"窗口 + 打开"新建列"窗口 """ def __init__(self, parent=None): @@ -1361,21 +1174,8 @@ class DataNewColumnForm(QWidget, DataNewColumn_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - -class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): +class DataMissingValueForm(DataProcessDialog, DataMissingValue_Ui_Form): """ 打开"数据-缺失值"窗口 """ @@ -1388,7 +1188,7 @@ class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): self.current_dataset = pd.DataFrame() # 当前数据集 self.current_dataset_name = "" - self.all_dataset = dict() + self.data_manager = dict() self.missing_dataset = pd.DataFrame() # 处理后的缺失值数据 self.missing_stat_dataset = pd.DataFrame() # 缺失值统计数据 @@ -1406,28 +1206,7 @@ class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): self.listWidget_selected.itemChanged.connect(self.dataset_filter_column) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def var_selected_del(self): current_row = self.listWidget_selected.currentRow() @@ -1578,10 +1357,10 @@ class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): def dataset_update(self): logging.info("发射导入数据信号") if len(self.missing_dataset) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(self.current_dataset_name, self.missing_dataset.to_dict(), path, create_time, update_time, remarks, file_size) # 发射信号 @@ -1598,10 +1377,10 @@ class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): if ok and (len(dataset_name) != 0): logging.info("发射导入数据信号") if len(self.missing_dataset) > 0: - create_time = self.all_dataset.get(self.current_dataset_name + '.create_time') + create_time = self.data_manager.get(self.current_dataset_name + '.create_time') update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 - path = self.all_dataset.get(self.current_dataset_name + '.path') - file_size = self.all_dataset.get(self.current_dataset_name + '.file_size') + path = self.data_manager.get(self.current_dataset_name + '.path') + file_size = self.data_manager.get(self.current_dataset_name + '.file_size') remarks = '' self.signal_data_change.emit(dataset_name, self.missing_dataset.to_dict(), path, create_time, update_time, remarks, file_size) # 发射信号 @@ -1611,7 +1390,7 @@ class DataMissingValueForm(QDialog, DataMissingValue_Ui_Form): self.close() -class DataSortForm(QWidget, DataSort_Ui_Form): +class DataSortForm(DataProcessDialog, DataSort_Ui_Form): """ 打开"从sas导入"窗口 """ @@ -1621,21 +1400,8 @@ class DataSortForm(QWidget, DataSort_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - - -class DataTransposeForm(QDialog, DataTranspose_Ui_Form): +class DataTransposeForm(DataProcessDialog, DataTranspose_Ui_Form): """ 打开"数据-转置"窗口 """ @@ -1668,29 +1434,6 @@ class DataTransposeForm(QDialog, DataTranspose_Ui_Form): self.pushButton_add.clicked.connect(self.var_selected_add) self.pushButton_delete.clicked.connect(self.var_selected_add) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") - # ================================自定义功能函数========================= def var_selected_del(self): current_row = self.listWidget_selected.currentRow() @@ -1794,7 +1537,7 @@ class DataTransposeForm(QDialog, DataTranspose_Ui_Form): self.close() -class DataStandardForm(QWidget, DataStandard_Ui_Form): +class DataStandardForm(DataProcessDialog, DataStandard_Ui_Form): """ 打开"从sas导入"窗口 """ @@ -1804,21 +1547,8 @@ class DataStandardForm(QWidget, DataStandard_Ui_Form): self.setupUi(self) self.center() - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - -class DataSampleForm(QDialog, DataSample_Ui_Form): +class DataSampleForm(DataProcessDialog, DataSample_Ui_Form): """ 打开"数据抽样"窗口 """ @@ -1851,29 +1581,6 @@ class DataSampleForm(QDialog, DataSample_Ui_Form): self.pushButton_weight_down.clicked.connect(self.var_weight_down) self.pushButton_weight_del.clicked.connect(self.var_weight_del) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") - def var_selected_del(self): # 移除选中的item current_row = self.listWidget_selected.currentRow() @@ -2052,7 +1759,7 @@ class DataSampleForm(QDialog, DataSample_Ui_Form): self.close() -class DataColumnNameForm(QDialog, DataColumnName_Ui_Form): +class DataColumnNameForm(DataProcessDialog, DataColumnName_Ui_Form): """ 打开"数据-列名处理"窗口 """ @@ -2348,7 +2055,7 @@ class DataColumnEncodeForm(QWidget, DataColumnEncode_Ui_Form): self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) -class DataReplaceForm(QDialog, DataReplace_Ui_Form): +class DataReplaceForm(DataProcessDialog, DataReplace_Ui_Form): """ 打开"数据-内容替换"窗口 """ @@ -2375,28 +2082,7 @@ class DataReplaceForm(QDialog, DataReplace_Ui_Form): self.pushButton_find.clicked.connect(self.data_find) self.pushButton_replace.clicked.connect(self.dataset_update) - def keyPressEvent(self, e): - """ - 按键盘Escape退出当前窗口 - @param e: - """ - if e.key() == Qt.Key_Escape: - self.close() - - def center(self): - screen = QDesktopWidget().screenGeometry() - size = self.geometry() - self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) - # ================================自定义槽函数========================= - def get_help(self): - """ - 打开帮助页面 - """ - try: - webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") - except Exception as e: - webbrowser.open_new_tab("http://www.py2cn.com") def ui_init(self): diff --git a/pyminer/features/sample/io.py b/pyminer/features/sample/io.py index 85f587d9924efaabd9303cbd902c4faed51ec165..cb3aae5606bd13cb85efffdc7a455b4dfe0a5563 100644 --- a/pyminer/features/sample/io.py +++ b/pyminer/features/sample/io.py @@ -11,7 +11,7 @@ from PyQt5 import QtWidgets, QtCore from PyQt5.QtWidgets import * from PyQt5.QtCore import * -#导入功能组件 +# 导入功能组件 from pyminer.ui.data.data_import_text import Ui_Form as Import_Ui_Form from pyminer.ui.data.data_import_excel import Ui_Form as ExcelImport_Ui_Form from pyminer.ui.data.data_import_spss import Ui_Form as SPSSImport_Ui_Form @@ -23,7 +23,7 @@ logging.basicConfig(format="%(asctime)s %(name)s:%(levelname)s:%(message)s", dat level=logging.INFO) -class ImportForm(QDialog,Import_Ui_Form): +class ImportForm(QDialog, Import_Ui_Form): """ "导入"窗口 """ @@ -34,7 +34,7 @@ class ImportForm(QDialog,Import_Ui_Form): self.setupUi(self) self.center() - #QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) + # QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) self.file_path = '' self.current_dataset_name = "" @@ -104,9 +104,9 @@ class ImportForm(QDialog,Import_Ui_Form): self.file_path, header, skiprows, nrows_preview, na_values)) self.current_dataset = pd.read_csv(self.file_path, engine="python", sep=sep, - encoding=encoding, - header=header, - skiprows=skiprows, nrows=nrows, na_values=na_values) + encoding=encoding, + header=header, + skiprows=skiprows, nrows=nrows, na_values=na_values) if len(self.current_dataset) == 0: self.tableWidget_previewData.clear() @@ -124,8 +124,8 @@ class ImportForm(QDialog,Import_Ui_Form): update_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') # 数据更新时间 path = self.file_path print("path:", path) - file_size=str(os.path.getsize(path)) - print("file_size:",file_size) + file_size = str(os.path.getsize(path)) + print("file_size:", file_size) remarks = '' self.signal_data_change.emit(self.current_dataset_name, self.current_dataset.to_dict(), path, create_time, update_time, remarks, file_size) # 发射信号 @@ -195,8 +195,8 @@ class ImportForm(QDialog,Import_Ui_Form): na_values = self.lineEdit_missValue.text() self.current_dataset = pd.read_csv(self.file_path, engine="python", sep=sep, encoding=encoding, - header=header, - skiprows=skiprows, nrows=nrows, na_values=na_values) + header=header, + skiprows=skiprows, nrows=nrows, na_values=na_values) if len(self.current_dataset) == 0: self.tableWidget_previewData.clear() @@ -242,7 +242,7 @@ class ImportForm(QDialog,Import_Ui_Form): self.tableWidget_previewData.setItem(i, j, newItem) -class ImportDatabase(QDialog,ImportDatabase_Ui_Form): +class ImportDatabase(QDialog, ImportDatabase_Ui_Form): """ "导入"窗口 """ @@ -256,14 +256,14 @@ class ImportDatabase(QDialog,ImportDatabase_Ui_Form): self.current_dataset_name = '' # 当前数据集名称 self.current_dataset = pd.DataFrame() # 修改后的数据集 - self.all_dataset='' #当前已导入的全部数据 - self.file_path='' #当前数据路径 + self.data_manager = None # 当前已导入的全部数据 + self.file_path = '' # 当前数据路径 - #QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) + # QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) self.label_test.setHidden(True) self.lineEdit_passwd.setEchoMode(QtWidgets.QLineEdit.Password) - #事件 + # 事件 self.pushButton_cancel.clicked.connect(self.close) self.pushButton_test.clicked.connect(self.database_test) self.pushButton_ok.clicked.connect(self.import_send_dataset) @@ -291,21 +291,20 @@ class ImportDatabase(QDialog,ImportDatabase_Ui_Form): logging.info("导入数据信号发射失败") self.close() - def database_test(self): import pymysql - host=self.lineEdit_host.text() - user=self.lineEdit_user.text() - passwd=self.lineEdit_passwd.text() - db=self.lineEdit_db.text() - port=self.spinBox_port.value() + host = self.lineEdit_host.text() + user = self.lineEdit_user.text() + passwd = self.lineEdit_passwd.text() + db = self.lineEdit_db.text() + port = self.spinBox_port.value() charset = 'utf8' - table=self.lineEdit_table.text() - sql='select * from '+db+'.'+table; + table = self.lineEdit_table.text() + sql = 'select * from ' + db + '.' + table; print(sql) - if len(db)==0: - QMessageBox.information(self, '注意', '数据库名不能为空', QMessageBox.Ok , QMessageBox.Ok) - elif len(table)==0: + if len(db) == 0: + QMessageBox.information(self, '注意', '数据库名不能为空', QMessageBox.Ok, QMessageBox.Ok) + elif len(table) == 0: QMessageBox.information(self, '注意', '表名不能为空', QMessageBox.Ok, QMessageBox.Ok) else: try: @@ -326,25 +325,25 @@ class ImportDatabase(QDialog,ImportDatabase_Ui_Form): def database_connect(self): import pymysql - host=self.lineEdit_host.text() - user=self.lineEdit_user.text() - passwd=self.lineEdit_passwd.text() - db=self.lineEdit_db.text() - port=self.spinBox_port.value() + host = self.lineEdit_host.text() + user = self.lineEdit_user.text() + passwd = self.lineEdit_passwd.text() + db = self.lineEdit_db.text() + port = self.spinBox_port.value() charset = 'utf8' - table=self.lineEdit_table.text() - self.current_dataset_name=table - self.file_path=db+'.'+table - sql='select * from '+self.file_path; + table = self.lineEdit_table.text() + self.current_dataset_name = table + self.file_path = db + '.' + table + sql = 'select * from ' + self.file_path; print(sql) - if len(db)==0: - QMessageBox.information(self, '注意', '数据库名不能为空', QMessageBox.Ok , QMessageBox.Ok) - elif len(table)==0: + if len(db) == 0: + QMessageBox.information(self, '注意', '数据库名不能为空', QMessageBox.Ok, QMessageBox.Ok) + elif len(table) == 0: QMessageBox.information(self, '注意', '表名不能为空', QMessageBox.Ok, QMessageBox.Ok) else: try: conn = pymysql.connect(host=host, user=user, passwd=passwd, db=db, port=port, charset=charset) - df = pd.read_sql(sql, con = conn) + df = pd.read_sql(sql, con=conn) return df conn.close() print('连接成功') @@ -375,7 +374,7 @@ class ImportDatabase(QDialog,ImportDatabase_Ui_Form): self.close() -class ImportExcelForm(QDialog,ExcelImport_Ui_Form): +class ImportExcelForm(QDialog, ExcelImport_Ui_Form): """ 打开"从excel导入"窗口 """ @@ -386,9 +385,9 @@ class ImportExcelForm(QDialog,ExcelImport_Ui_Form): self.setupUi(self) self.center() - self.file_path = '' #文件路径 - self.current_dataset_name = "" #当前数据集名称 - self.current_dataset = pd.DataFrame() #当前数据集 + self.file_path = '' # 文件路径 + self.current_dataset_name = "" # 当前数据集名称 + self.current_dataset = pd.DataFrame() # 当前数据集 # 导入窗口的相关事件 # 在"导入"窗口,打开选择文件 @@ -489,8 +488,8 @@ class ImportExcelForm(QDialog,ExcelImport_Ui_Form): self.file_path, sheet, header, skiprows, nrows_preview, na_values)) self.current_dataset = pd.read_excel(self.file_path, sheet_name=sheet, - header=header, - skiprows=skiprows, nrows=nrows, na_values=na_values) + header=header, + skiprows=skiprows, nrows=nrows, na_values=na_values) if len(self.current_dataset) == 0: self.tableWidget_previewData.clear() @@ -563,7 +562,7 @@ class ImportExcelForm(QDialog,ExcelImport_Ui_Form): logging.info("file_path:{},sheet_name:{},header:{},skiprows:{},nrows:{},na_values:{}".format( self.file_path, sheet, header, skiprows, nrows_preview, na_values)) - if len(sheet)>0: + if len(sheet) > 0: self.current_dataset = pd.read_excel(self.file_path, sheet_name=sheet, header=header, skiprows=skiprows, nrows=nrows, na_values=na_values) @@ -612,8 +611,7 @@ class ImportExcelForm(QDialog,ExcelImport_Ui_Form): self.tableWidget_previewData.setItem(i, j, newItem) - -class ImportSpssForm(QDialog,SPSSImport_Ui_Form): +class ImportSpssForm(QDialog, SPSSImport_Ui_Form): """ 打开"从spss导入"窗口 """ @@ -624,7 +622,7 @@ class ImportSpssForm(QDialog,SPSSImport_Ui_Form): self.setupUi(self) self.center() - #QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) + # QssTools.set_qss_to_obj(ui_dir + "/source/qss/patata.qss", self) self.file_path = '' self.current_dataset_name = "" @@ -826,7 +824,7 @@ class ImportSpssForm(QDialog,SPSSImport_Ui_Form): return self.current_dataset.columns -class ImportSasForm(QDialog,SASImport_Ui_Form): +class ImportSasForm(QDialog, SASImport_Ui_Form): """ 打开"从sas导入"窗口 """ diff --git a/pyminer/pmapp.py b/pyminer/pmapp.py index d0820123bf08d5e3a9b337888b4d9ffa387d6467..90e427f56868ccd65d4ec13d19d7ced8264057bc 100644 --- a/pyminer/pmapp.py +++ b/pyminer/pmapp.py @@ -9,13 +9,12 @@ import webbrowser import qdarkstyle import os, sys -from PyQt5.QtCore import QSize, pyqtSignal, QTimer, QRectF -from PyQt5.QtWidgets import QApplication, QStyleFactory, QDesktopWidget, QMessageBox, QSplashScreen, QTreeWidgetItem, \ - QListWidgetItem, QWidget, QHBoxLayout, QGridLayout, QLabel, QSpacerItem, QSizePolicy, \ - QGraphicsDropShadowEffect, QListWidget, QGraphicsScene, QAbstractItemView, QTableWidgetItem, QGraphicsObject, \ +from PyQt5.QtCore import QRectF +from PyQt5.QtWidgets import QApplication, QStyleFactory, QDesktopWidget, QMessageBox, QTreeWidgetItem, \ + QGraphicsScene, QAbstractItemView, QTableWidgetItem, QGraphicsObject, \ QGraphicsItem, QFileDialog from PyQt5.QtWidgets import QMainWindow, QAction, qApp, QMenu -from PyQt5.QtGui import QIcon, QPixmap, QImage, QColor, QPainter, QPainterPath, QPen, QTextCursor, QBrush, QCursor +from PyQt5.QtGui import QIcon, QColor, QPainter, QPen, QTextCursor, QBrush, QCursor from pyminer.features.sample import io from pyminer.ui.base.mainForm import Ui_MainWindow @@ -41,6 +40,7 @@ import pandas as pd from pyminer.ui.base.widgets.notificationwidget import NotificationWindow from pyminer.features.statistics.test_basic_stats import StatsBaseForm +from pyminer.share.datahost import DataManager TextStyle = """ QMessageBox QPushButton[text="OK"] { @@ -102,6 +102,7 @@ QMessageBox QPushButton[text="Ignore"] { } """ + class Application(QApplication): def __init__(self, argv): QApplication.__init__(self, argv) @@ -132,9 +133,6 @@ class Application(QApplication): self.setStyle(QStyleFactory.create("Windows")) - - - class CustomRect(QGraphicsObject): def __init__(self): super(CustomRect, self).__init__() @@ -186,11 +184,16 @@ class MyMainForm(QMainWindow, Ui_MainWindow): self.center() self.__file_path_choose = "" - self.__all_dataset = dict() # 定义字典,保存全部数据集的基本信息 - self.__all_dataset_name = set() # 定义集合,保存全部数据集名称,确保不会重复 + # self.__all_dataset = dict() # 定义字典,保存全部数据集的基本信息 + # self.__all_dataset_name = set() # 定义集合,保存全部数据集名称,确保不会重复 + + self.data_manager = DataManager() # __all_dataset和__all_dataset_name都已经注释掉了。现在用self.datahost来代替。 + # 关于datahost的用法见其定义的文件。 + self.__current_dataset_name = "" self.__current_dataset = pd.DataFrame() self.__current_dataset_dtype = set() # 定义集合,保存当前数据集中存在的数据类型,确保不会重复 + self.__result_path = '' # 结果文件保存路径 self.__setting = dict() # 设置文件 self.slot_flush_console('info', 'system', 'pyminer已准备就绪') @@ -740,7 +743,6 @@ class MyMainForm(QMainWindow, Ui_MainWindow): """ print("开始更新数据") node_name = self.treeWidget_storehouse.currentItem().text(0) - print(node_name) all_data = pd.DataFrame(self.__all_dataset.get(node_name)) # 获取当前数据 self.__current_dataset_name = node_name # 修改当前数据名称 # 获取已经导入页面获取的数据集 @@ -832,7 +834,7 @@ class MyMainForm(QMainWindow, Ui_MainWindow): # 修改当前正在使用的数据集 self.__current_dataset = dataset self.__current_dataset_name = dataset_name - self.__all_dataset_name.add(dataset_name) # 保存数据集名称 + row = str(dataset.shape[0]) # 行数 col = str(dataset.shape[1]) # 列数 memory_size = self.__current_dataset.memory_usage().sum() @@ -867,33 +869,12 @@ class MyMainForm(QMainWindow, Ui_MainWindow): data_notna_cnt.append(dataset.loc[:, i].notna().sum()) info = pd.DataFrame({"变量名称": list(dataset.columns), "数据类型": data_type, "非空值数量": data_notna_cnt}) - dataset_new = {dataset_name: dataset, - dataset_name + ".path": path, - dataset_name + ".create_time": create_time, - dataset_name + ".update_time": update_time, - dataset_name + ".row": row, - dataset_name + ".col": col, - dataset_name + ".remarks": remarks, - dataset_name + ".file_size": file_usage, - dataset_name + ".memory_usage": memory_usage, - dataset_name + ".info": info, } - # 添加当前数据集到数据集列表中 print("添加当前数据集到数据集列表中") - # 检查数据集名称是否已存在,如果存在则修改相关信息,否则进行新增 - if self.__current_dataset_name in list(self.__all_dataset.keys()): - self.__all_dataset[dataset_name] = dataset - self.__all_dataset[dataset_name + ".path"] = path - self.__all_dataset[dataset_name + ".create_time"] = create_time - self.__all_dataset[dataset_name + ".update_time"] = update_time - self.__all_dataset[dataset_name + ".row"] = row - self.__all_dataset[dataset_name + ".col"] = col - self.__all_dataset[dataset_name + ".remarks"] = remarks - self.__all_dataset[dataset_name + ".file_size"] = file_size - self.__all_dataset[dataset_name + ".memory_usage"] = memory_usage - self.__all_dataset[dataset_name + ".info"] = info - else: - self.__all_dataset.update(dataset_new) + self.data_manager.set_data(dataset_name, dataset) + self.data_manager.set_info(dataset_name, path=path, create_time=create_time, update_time=update_time, + row=row, col=col, remarks=remarks, file_size=file_size, memory_usage=memory_usage, + info=info) self.tabWidget.setCurrentIndex(0) def add_dataset_to_workdir(self, dataset_name): @@ -934,7 +915,7 @@ class MyMainForm(QMainWindow, Ui_MainWindow): 显示"从数据库导入"窗口 """ self.import_database = io.ImportDatabase() - self.import_database.all_dataset = self.__all_dataset + self.import_database.data_manager = self.data_manager self.import_database.signal_data_change.connect(self.slot_dataset_reload) self.import_database.show() @@ -942,10 +923,10 @@ class MyMainForm(QMainWindow, Ui_MainWindow): """ 显示"数据-行筛选"窗口 """ - self.data_row_filter_form = preprocess.DataRowFilterForm() + self.data_row_filter_form = preprocess.DataRowFilterForm(data_manager=self.data_manager) self.data_row_filter_form.current_dataset = self.__current_dataset self.data_row_filter_form.current_dataset_name = self.__current_dataset_name - self.data_row_filter_form.all_dataset = self.__all_dataset + self.data_row_filter_form.current_dataset_columns = self.__current_dataset.columns self.data_row_filter_form.comboBox_columns.addItems(list(self.__current_dataset.columns)) for col in self.__current_dataset.columns: @@ -960,21 +941,20 @@ class MyMainForm(QMainWindow, Ui_MainWindow): 显示"数据信息"窗口 """ self.data_info = preprocess.DataInfoForm() - self.data_info.all_dataset = self.__all_dataset - self.data_info.all_dataset_name = list(self.__all_dataset_name) + self.data_info.data_manager = self.data_manager # [TODO]这是什么? + self.data_info.all_dataset_name = list(self.data_manager.all_data_names) self.data_info.current_dataset_name = self.__current_dataset_name self.data_info.lineEdit_dataset_name.setText(self.__current_dataset_name) - self.data_info.lineEdit_path.setText(self.__all_dataset.get(self.__current_dataset_name + ".path")) - self.data_info.lineEdit_row.setText(self.__all_dataset.get(self.__current_dataset_name + ".row")) - self.data_info.lineEdit_col.setText(self.__all_dataset.get(self.__current_dataset_name + ".col")) - self.data_info.lineEdit_file_size.setText(self.__all_dataset.get(self.__current_dataset_name + ".file_size")) - self.data_info.lineEdit_memory_usage.setText( - self.__all_dataset.get(self.__current_dataset_name + ".memory_usage")) - self.data_info.lineEdit_create_time.setText( - self.__all_dataset.get(self.__current_dataset_name + ".create_time")) - self.data_info.lineEdit_update_time.setText( - self.__all_dataset.get(self.__current_dataset_name + ".update_time")) - self.data_info.info = self.__all_dataset.get(self.__current_dataset_name + ".info") + + info_dic = self.data_manager.get_info(self.__current_dataset_name) # 调用get_info方法可以获取数据的信息,全部信息以列表的形式返回。 + + self.data_info.lineEdit_path.setText(info_dic.get('path')) + self.data_info.lineEdit_row.setText(info_dic.get('row')) + self.data_info.lineEdit_col.setText(info_dic.get('col')) + self.data_info.lineEdit_file_size.setText(info_dic.get('file_size')) + self.data_info.lineEdit_memory_usage.setText(info_dic.get('memory_usage')) + self.data_info.lineEdit_create_time.setText(info_dic.get('create_time')) + self.data_info.lineEdit_update_time.setText(info_dic.get('update_time')) self.data_info.info_init() self.data_info.exec_() @@ -991,7 +971,7 @@ class MyMainForm(QMainWindow, Ui_MainWindow): """ self.data_role = preprocess.DataRoleForm() self.data_role.current_dataset = self.__current_dataset.copy() - self.data_role.all_dataset = self.__all_dataset + self.data_role.data_manager = self.data_manager self.data_role.current_dataset_name = self.__current_dataset_name self.data_role.current_dataset_columns = self.__current_dataset.columns self.data_role.comboBox_columns.addItems(list(self.__current_dataset.columns)) @@ -1004,9 +984,9 @@ class MyMainForm(QMainWindow, Ui_MainWindow): 显示"数据-纵向合并"窗口 """ self.data_merge_vertical = preprocess.DataMergeVerticalForm() - self.data_merge_vertical.listWidget_dataset.addItems(list(self.__all_dataset_name)) - self.data_merge_vertical.all_dataset = self.__all_dataset - self.data_merge_vertical.all_dataset_name = self.__all_dataset_name + self.data_merge_vertical.listWidget_dataset.addItems(list(self.data_manager.all_data_names)) + self.data_merge_vertical.data_manager = self.data_manager + self.data_merge_vertical.all_dataset_name = self.data_manager.all_data_names self.data_merge_vertical.listWidget_start.addItem(self.__current_dataset_name) self.data_merge_vertical.current_dataset_name = self.__current_dataset_name self.data_merge_vertical.signal_data_change.connect(self.slot_dataset_reload) @@ -1017,11 +997,11 @@ class MyMainForm(QMainWindow, Ui_MainWindow): 显示"数据-横向合并"窗口 """ self.data_merge_horizontal = preprocess.DataMergeHorizontalForm() - self.data_merge_horizontal.listWidget_dataset.addItems(list(self.__all_dataset_name)) - self.data_merge_horizontal.all_dataset = self.__all_dataset - self.data_merge_horizontal.all_dataset_name = self.__all_dataset_name + self.data_merge_horizontal.listWidget_dataset.addItems(list(self.data_manager.all_data_names)) + self.data_merge_horizontal.data_manager = self.data_manager + self.data_merge_horizontal.all_dataset_name = self.data_manager.all_data_names self.data_merge_horizontal.listWidget_start.addItem(self.__current_dataset_name) - self.data_merge_vertical.current_dataset_name = self.__current_dataset_name + self.data_merge_horizontal.current_dataset_name = self.__current_dataset_name self.data_merge_horizontal.signal_data_change.connect(self.slot_dataset_reload) self.data_merge_horizontal.exec_() @@ -1050,7 +1030,7 @@ class MyMainForm(QMainWindow, Ui_MainWindow): """ self.data_missing_value = preprocess.DataMissingValueForm() self.data_missing_value.current_dataset = self.__current_dataset - self.data_missing_value.all_dataset = self.__all_dataset + self.data_missing_value.data_manager = self.data_manager self.data_missing_value.current_dataset_name = self.__current_dataset_name self.data_missing_value.current_dataset_columns = self.__current_dataset.columns self.data_missing_value.listWidget_var.addItems(list(self.__current_dataset.columns)) diff --git a/pyminer/pmstart.py b/pyminer/pmstart.py index b2fc416237490862556a7dc942e7243ad81480d4..c89ad6d59ae579e4ab446cb1069da6a4b8b24c70 100644 --- a/pyminer/pmstart.py +++ b/pyminer/pmstart.py @@ -30,7 +30,7 @@ def boot(): myWin.load_data(splash) pmutil._main_window = myWin - #myWin.data_import_file_test('class.csv') + myWin.data_import_file_test('class.csv') # 窗口美化设置 # QssTools.set_qss_to_obj(root_dir + "/ui/source/qss/pyminer.qss", app) diff --git a/pyminer/share/datahost/__init__.py b/pyminer/share/datahost/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b15a7830fd6c0c3381a1efd99801ff3386a3eb28 --- /dev/null +++ b/pyminer/share/datahost/__init__.py @@ -0,0 +1 @@ +from .datamanager import DataManager diff --git a/pyminer/share/datahost/datamanager.py b/pyminer/share/datahost/datamanager.py new file mode 100644 index 0000000000000000000000000000000000000000..f78acb5474ada498c503e54af850c06d70908fb3 --- /dev/null +++ b/pyminer/share/datahost/datamanager.py @@ -0,0 +1,325 @@ +import time +import warnings + +MAX_STEPS = 15 +import copy +from typing import Dict + + +class Data(): + ''' + 这是一个数据对象,可以管理数据的历史。 + 支持的对象类型应当是有限的,比如dataframe,ndarray等。 + 目前使用的就是这个类。 + ''' + value = None + info = {} + + def __init__(self, ini_value: object): + self.info = {} + self.set_data(ini_value) + + def set_data(self, value: object): + ''' + 添加数据。 + :param value: + :return: + ''' + self.value = value + # self.update_info() + + def get_data(self, copy_val=True) -> object: + ''' + 获取数据的值,采用当前的指针。 + :param copy_val: + 如果为True(默认),就返回复制后的对象。反之返回引用。当需要修改对象的时候,请保持True。 + 特别的,不要使用类似如下的方式。这样会多次复制对象,从而导致不必要的开销: + for i in range(10): + Data.get('a')[i] + 可以使用如下方案: + for i in range(10): + Data.get('a',copy=False)[i] + 但还是推荐使用: + a = Data.get('a') + for i in range(10): + a[i] + :return:object + ''' + return self.value + + def update_info(self): # 目前逻辑是写死了的。 + name = 'test_file' + path = '../class.csv' + create_time = '2017.9.1' # self.info['create_time'] + update_time = '时间戳:' + str(time.time()) + row = '10' + col = '10' + remarks = '' + file_size = '123' + memory_usage = '123kb' + info = '' + self.set_info(name, path, create_time, update_time, row, col, remarks, file_size, memory_usage, info) + + def set_info(self, dataset_name: str, path: str, create_time: str, + update_time: str, row: str, col: str, remarks: str, file_size: str, + memory_usage: str, info=''): + dataset_info = {"name": dataset_name, + "path": path, + "create_time": create_time, + "update_time": update_time, + "row": row, + "col": col, + "remarks": remarks, + "file_size": file_size, + "memory_usage": memory_usage, + "info": info} + self.info = dataset_info + + +class Data_With_Undo_Func(): + ''' + 这是一个数据对象,可以管理数据的历史。 + 支持的对象类型应当是有限的,比如dataframe,ndarray等。 + 目前用不上这个类,因为这个比较复杂。 + ''' + + def __init__(self, ini_value: object): + self.value_stack = [] + self.stack_pointer = 0 # 指向起始位置的指针。以下代码注释中简称‘指针’ + self.info = {} + self.set_data(ini_value) + + def __len__(self): + return len(self.value_stack) + + def get_latest(self, copy_val=True) -> object: + if copy_val: + return copy.copy(self.value_stack[-1]) + return self.value_stack[-1] + + def set_data(self, value: object): + ''' + 添加数据。 + :param value: + :return: + ''' + self.value_stack.append(value) + while len(self.value_stack) > MAX_STEPS: + self.value_stack.pop(0) + self.stack_pointer = len(self) - 1 + # self.update_info() + + def get_data(self, copy_val=True) -> object: + ''' + 获取数据的值,采用当前的指针。 + :param copy_val: + 如果为True(默认),就返回复制后的对象。反之返回引用。当需要修改对象的时候,请保持True。 + 特别的,不要使用类似如下的方式。这样会多次复制对象,从而导致不必要的开销: + for i in range(10): + Data.get('a')[i] + 可以使用如下方案: + for i in range(10): + Data.get('a',copy=False)[i] + 但还是推荐使用: + a = Data.get('a') + for i in range(10): + a[i] + :return:object + ''' + index = self.stack_pointer + if copy_val: + return copy.copy(self.value_stack[index]) + return self.value_stack[index] + + def get_index(self) -> int: + return self.stack_pointer + + def set_index(self, index: int): + if 0 <= index < len(self.value_stack): + self.stack_pointer = index + + def update_info(self): # 目前逻辑是写死了的。 + name = 'test_file' + path = '../class.csv' + create_time = '2017.9.1' # self.info['create_time'] + update_time = '时间戳:' + str(time.time()) + row = '10' + col = '10' + remarks = '' + file_size = '123' + memory_usage = '123kb' + info = '' + self.set_info(name, path, create_time, update_time, row, col, remarks, file_size, memory_usage, info) + + def set_info(self, dataset_name: str, path: str, create_time: str, + update_time: str, row: str, col: str, remarks: str, file_size: str, + memory_usage: str, info=''): + + dataset_info = {"name": dataset_name, + "path": path, + "create_time": create_time, + "update_time": update_time, + "row": row, + "col": col, + "remarks": remarks, + "file_size": file_size, + "memory_usage": memory_usage, + "info": info} + self.info = dataset_info + + +class DataManager(): + ''' + 以下举例均默认data_manager = DataManager() + ''' + + def __init__(self): + self.vars: Dict[str, Data] = {} + self.recycle_bin: Dict[str, Data] = {} + self.current_data = None + # self.all_data_sets_info: Dict[str, dict] = {} # 管理数据集的信息 + + @property + def all_data_names(self) -> set: + ''' + 调用方式:data_manager.all_data_names + 可以代替__all_data_names + Returns: + + ''' + return {k for k in self.vars.keys()} + + def get_info(self, name: str): + ''' + + Args: + name: 变量的名称 + + Returns:字典。返回变量的全部属性字典。比如get_info('class')['path'],就是返回名为‘class’的数据集中的‘path’属性。 + + ''' + return self.vars[name].info + + def set_data(self, name: str, value: object) -> None: + ''' + 设置信息。 + 比如说,set_data('a'),就是设置数据集a的值。 + Args: + name: + value: + + Returns: + + ''' + if name in self.vars.keys(): + self.vars[name].set_data(value) + else: + self.vars[name] = Data(value) + + def get_all_data_dic(self): + ''' + 获取所有数据,是一个名称对元素的字典。但是要注意,返回的值是一个Data对象,而不是变量的字典。 + Returns: + + ''' + return self.vars + + def get_data(self, name: str, copy_val: bool = False) -> object: + ''' + 获取名为name的data。如果copy_val是True,那么就将其设置为True之后再 + Args: + name: + copy_val: + + Returns: + + ''' + if name in self.vars.keys(): + return self.vars[name].get_data(copy_val=copy_val) + else: + warnings.warn("Variable \'%s\' isn\'t exist." % name) + + def get(self, name: str) -> object: + ''' + 等于get_data。 + Args: + name: + + Returns: + + ''' + return self.get_data(name) + + def move_data_stack_pointer(self, name: str, step: int = 1): + ''' + 无效方法,暂时无用。等到引入撤销功能之后就用上了。 + Args: + name: + step: + + Returns: + + ''' + if name not in self.vars.keys(): + return + var = self.vars[name] + pos = var.stack_pointer + step + if 0 <= pos < len(var): + var.stack_pointer = pos + elif pos < 0: + var.stack_pointer = 0 + print('到达最初数据,不可再撤销!') + elif pos >= len(var): + var.stack_pointer = len(var) - 1 + print('到达最新数据,不可再重做!') + + def move_data_stack_pointer_to(self, name: str, pos: int = -1): + ''' + 暂时用不上的方法。 + Args: + name: + pos: + + Returns: + + ''' + if name not in self.vars.keys(): + return + var = self.vars[name] + if pos < 0: + pos = len(var.value_stack) - pos + + if 0 <= pos < len(var): + var.stack_pointer = pos + elif pos < 0: + var.stack_pointer = 0 + print('到达最初数据,不可再撤销!') + elif pos >= len(var): + var.stack_pointer = len(var) - 1 + print('到达最新数据,不可再重做!') + + def delete_data(self, name: str) -> None: + '''删除数据,也就是移动到回收站。''' + if name in self.vars.keys(): + self.recycle_bin[name] = self.vars[name] + self.vars.pop(name) + else: + warnings.warn("Variable \'%s\' isn\'t exist." % name) + + def recover_data(self, name: str) -> None: + '''从回收站中恢复数据''' + if name in self.recycle_bin.keys(): + self.vars[name] = self.recycle_bin[name] + self.recycle_bin.pop(name) + else: + warnings.warn("Variable \'%s\' isn\'t exist." % name) + + def set_info(self, dataset_name: str, path: str, create_time: str, + update_time: str, row: str, col: str, remarks: str, file_size: str, + memory_usage: str, info=''): + '''设置信息''' + + self.vars[dataset_name].set_info(dataset_name, path=path, create_time=create_time, update_time=update_time, + row=row, col=col, remarks=remarks, file_size=file_size, + memory_usage=memory_usage, + info=info) diff --git a/pyminer/ui/base/widgets/controlpanel.py b/pyminer/ui/base/widgets/controlpanel.py index 640de261e595c35bc8c86f6e332a342ab6a7a627..7f5bd876ebc2797d1d4bdfcf98bbffe10f6297b8 100644 --- a/pyminer/ui/base/widgets/controlpanel.py +++ b/pyminer/ui/base/widgets/controlpanel.py @@ -274,7 +274,7 @@ class PMPageData(QWidget): # 删除行 self.btn_delete_row.clicked.connect(app.data_delete_row_display) - self.btn_delete_row.clicked.connect(app.data_delete_col_display) + self.btn_delete_column.clicked.connect(app.data_delete_col_display) # 纵向合并 self.btn_data_merge_vertical.clicked.connect(app.data_merge_vertical_display) diff --git a/pyminer/ui/base/widgets/generalwidgets/table/tableviews.py b/pyminer/ui/base/widgets/generalwidgets/table/tableview.py similarity index 100% rename from pyminer/ui/base/widgets/generalwidgets/table/tableviews.py rename to pyminer/ui/base/widgets/generalwidgets/table/tableview.py diff --git a/pyminer/ui/base/widgets/generalwidgets/table/tablewidgets.py b/pyminer/ui/base/widgets/generalwidgets/table/tablewidget.py similarity index 100% rename from pyminer/ui/base/widgets/generalwidgets/table/tablewidgets.py rename to pyminer/ui/base/widgets/generalwidgets/table/tablewidget.py diff --git a/pyminer/ui/base/widgets/generalwidgets/window/__init__.py b/pyminer/ui/base/widgets/generalwidgets/window/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a2ecdc1ce8eaa0efad1c74ed2b9492e65c23f9 --- /dev/null +++ b/pyminer/ui/base/widgets/generalwidgets/window/__init__.py @@ -0,0 +1 @@ +from .dialog import DataProcessDialog \ No newline at end of file diff --git a/pyminer/ui/base/widgets/generalwidgets/window/dialog.py b/pyminer/ui/base/widgets/generalwidgets/window/dialog.py new file mode 100644 index 0000000000000000000000000000000000000000..b9f7515113b6aba3399228cdc42fa42b687eea71 --- /dev/null +++ b/pyminer/ui/base/widgets/generalwidgets/window/dialog.py @@ -0,0 +1,32 @@ +from PyQt5.QtWidgets import QDialog, QDesktopWidget +from PyQt5.QtCore import Qt + +import webbrowser + + +class DataProcessDialog(QDialog): + ''' + 用于数据处理的对话框。特点是按下esc之后会退出等。 + ''' + + def keyPressEvent(self, e): + """ + 按键盘Escape退出当前窗口 + @param e: + """ + if e.key() == Qt.Key_Escape: + self.close() + + def center(self): + screen = QDesktopWidget().screenGeometry() + size = self.geometry() + self.move(int((screen.width() - size.width()) / 2), int((screen.height() - size.height()) / 2)) + + def get_help(self): + """ + 打开帮助页面 + """ + try: + webbrowser.get('chrome').open_new_tab("http://www.py2cn.com") + except Exception as e: + webbrowser.open_new_tab("http://www.py2cn.com")