from openpyxl import Workbook, load_workbook from dataclasses import dataclass from pprint import pprint as print from table_handler.utils import hex_to_rgb, delegate_tasks from logger import lock_logger from .commands.find_and_replace import FindAndReplace from .commands.regex_edit_col import RegexEditCol from .commands.regex_move_to_col import RegexMoveToCol from .commands.copy import Copy from .commands.regex_group_valuebased_choose import RegexGroupValuebasedChoose from .commands.row_color_to_value import RowColorToCalue from .conditions.regex_cell import RegexCellCondition from multiprocessing import Pool, Queue, Process, Manager, cpu_count @dataclass class Entry: value: str|int|float|bool|None bg_color: tuple[str, str, str] fg_color: tuple[str, str, str] class XslxReader: def __init__(self, file): self.wb: Workbook = load_workbook(file, read_only=True) self.ws = self.wb.active if not self.ws: raise ValueError("No active worksheet found in the workbook.") def serialize_table(self, expected_cols: list[str]) -> dict[int, list[Entry]]: data = {col: {} for col in expected_cols} map_expected_with_ws = {} for idx, cell in enumerate(self.ws[1]): cell_value = cell.value if cell.value else "" if cell_value in expected_cols: map_expected_with_ws[cell_value] = idx else: print(f"Skipping unexpected Col {cell_value}") if set(map_expected_with_ws.keys()) != set(expected_cols): raise ValueError("Missing Col in input, check your table header") for rid, row in enumerate(self.ws.iter_rows(min_row=2, values_only=False), start=2): for expected_col, ws_idx in map_expected_with_ws.items(): cell = row[ws_idx] data[expected_col][rid] = Entry( value=str(cell.value) if not cell.value is None else "", bg_color=hex_to_rgb(cell.fill.start_color.rgb if cell.fill and cell.fill.start_color else "#0"), fg_color=hex_to_rgb(cell.font.color.rgb if cell.font and cell.font.color else "#0") ) return data class Transformer: def __init__(self, data, expected_output_schema: list[str], cumulated_instructions = {}): self.data = data self.fully_serialized_output = {} self.seperator = ", " self.expected_output_schema = expected_output_schema self.implicit_copy = True self.output_header = True self.cumulated_instructions = cumulated_instructions def write_output(self, destination_file: str): if not self.fully_serialized_output: raise ValueError("There is nothing to write, serialize first") # write header owb = Workbook() ows = owb.active if self.output_header: for col_i, col_name in enumerate(self.expected_output_schema, start=1): ows.cell(row=1, column=col_i, value=col_name) # write rows for row_i, (row_id, row_data) in enumerate(self.fully_serialized_output.items(), start=2 if self.output_header else 1): for col_i, col_name in enumerate(self.expected_output_schema, start=1): ows.cell(row=row_i, column=col_i, value=row_data.get(col_name)) owb.save(destination_file) def _check_conditions(self, instruct: dict[str, str], row: dict[str, any]): when_regex_condition_map = instruct.get("when_regex") if when_regex_condition_map: return RegexCellCondition(when_regex_condition_map).check_on_row(row) return True def _merge_change_to_cel(self, value, changes): #if not value and changes: # value = changes #elif changes: # value = f"{value}{self.seperator}{changes}" value = changes return value def _extecute_instruct(self, source_cell: str, instruct: dict, row: dict[str, any], working_row: dict[str, any]): # execute changes res = None res_row = {} match instruct.get("command"): case "FIND_AND_REPLACE": res = FindAndReplace(instruct["map"]).execute_on_cell(row[source_cell]) case "REGEX_MOVE_TO_COL": res_row = RegexMoveToCol(instruct["match"], instruct["to"], instruct.get("with_input_iteration")) \ .execute_on_row(row, source_cell) case "REGEX_EDIT_COL": res = RegexEditCol(instruct["match"], instruct["replace_with"], instruct.get("matching_group_rearange"))\ .execute_on_cell(row[source_cell]) case "REGEX_GROUP_VALUEBASED_CHOOS": res = RegexGroupValuebasedChoose(instruct["match"], instruct["highest"])\ .execute_on_cell(row[source_cell]) case "CONDITIONAL_COPY": res_row = Copy(instruct["source"], instruct["target"], replace=instruct.get("replace"))\ .execute_on_row(row) case "ROW_COLOR_TO_BOOL": res_row = RowColorToCalue(instruct["target"], instruct["value"], instruct.get("fg_colors"), instruct.get("bg_colors"), instruct.get("approximate"), instruct.get("threshold"))\ .execute_on_row(row) # Merge row_changes to working_row for target_cel, change in res_row.items(): working_row[target_cel] = self._merge_change_to_cel(working_row.get(target_cel), change) # Merge changes in working row if source_cell and res is not None: working_row[source_cell] = self._merge_change_to_cel(working_row.get(source_cell), res) return working_row def _run_row_resolution(self, range, worker: str = "X", q = None): self.fully_serialized_output = {} end = list(range)[-1] for rid in range: if rid % 100 == 0 and q: q.put(f"Worker {worker}:Working on row {rid}/{end}") working_row = {} source_row = self.get_row_as_dict(rid) for source_cols, instructions in self.cumulated_instructions: for instruct in instructions if type(instructions) == list else [instructions]: for col in source_cols: if not self._check_conditions(instruct, source_row): continue working_row = self._extecute_instruct(col, instruct, source_row, working_row) if not self.implicit_copy: continue for col in self. expected_output_schema: if col in working_row: continue working_row[col] = source_row.get(col).value self.fully_serialized_output[rid] = working_row return self.fully_serialized_output @property def vertical_len(self): return max(len(data_row) for data_row in self.data.values()) def get_row_as_dict(self, rid: int): return {col: rows.get(rid) for col, rows in self.data.items()} def run(self, col_instructions: dict[tuple[str], dict], row_instructions: dict[tuple[str], dict]): self.cumulated_instructions = list(col_instructions.items()) + [ ((None,), instr) for instr in row_instructions ] amount_working_rows = self.vertical_len + 2 with Manager() as manager: q = manager.Queue() chnucs = [[elem, worker_name, q] for worker_name, elem in enumerate( delegate_tasks(range(2, amount_working_rows), cpu_count(), 1000) )] log_proc = Process(target=lock_logger, args=(q,)) log_proc.start() with Pool(len(chnucs)) as p: result = p.starmap( Transformer(self.data, self.expected_output_schema, self.cumulated_instructions)._run_row_resolution, chnucs ) q.put("DONE") log_proc.join(1) self.fully_serialized_output = {k: v for d in result for k, v in d.items()} return