fixing things

.gitignore

@@ -1 +1,2 @@
 .venv
+/**/*.pyc

.vscode/launch.json

@@ -8,7 +8,7 @@
             "name": "Rings",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/__init__.py",
+            "module": "fledermaus_formatter",
             "args": ["-i", "data/ringe_short.xlsx", "-o", "ringe_short_out.xlsx", "-t", "ring"],
             "console": "integratedTerminal"
         },
@@ -16,7 +16,7 @@
             "name": "Countings",
             "type": "debugpy",
             "request": "launch",
-            "program": "${workspaceFolder}/__init__.py",
+            "module": "fledermaus_formatter",
             "args": ["-i", "data/zählungen_short.xlsx", "-o", "zählungen_short_out.xlsx", "-t", "counting"],
             "console": "integratedTerminal"
         }

README.md

@@ -0,0 +1,9 @@
+# Fledermaus Formater
+
+The core of this project is an interpreter which takes instructions, defined in [/instructions](/instructions/), and uses them to apply operations upon an xlsx table.
+For this was developed to serve the specific use case for formatting Bat Data, the instructions needed for said operation are monolithically included in this repository at this early version of the formatter.
+
+## Installing
+
+1. Download and run: [install.sh](install.sh)
+2. Execute the application using `fledermaus-formater` from the terminal; the app is self-documented

__init__.py

@@ -1,62 +0,0 @@
-import argparse
-from table_handler.serializer import Transformer, XslxReader
-from logger import logger 
-
-import instructions.countings_formater as countings_formater
-import instructions.ring_formater as ring_formater
-
-
-def main():
-
-    parser = argparse.ArgumentParser(description="Serialize Excel tables.")
-
-    parser.add_argument(
-        "-i", "--input",
-        required=True,
-        help="Input Excel file"
-    )
-
-    parser.add_argument(
-        "-o", "--output",
-        required=True,
-        help="Output Excel file"
-    )
-
-    parser.add_argument(
-        "-t", "--type",
-        required=True,
-        choices=["ring", "counting"],
-        help="Type of table to process"
-    )
-
-    args = parser.parse_args()
-
-    # Select formatter based on type
-    if args.type == "ring":
-        formatter = ring_formater
-        row_instructions = ring_formater.ROW_INSTRUCTIONS
-    else:
-        formatter = countings_formater
-        row_instructions = countings_formater.ROW_INSTRUCTION
-
-    logger.info("Reading file, building serializer...")
-    data = XslxReader(args.input).serialize_table(formatter.EXPECTED_INPUT_TABLE_SCHEMA)
-    
-    serializer = Transformer(
-        data,
-        formatter.EXPECTED_OUTPUT_TABLE_SCHEMA or formatter.EXPECTED_INPUT_TABLE_SCHEMA,
-    )
-
-    logger.info("Starting transformation")
-    serializer.run(
-        formatter.COL_INSTRUCTIONS,
-        row_instructions
-    )
-
-    logger.info("Writing transformations")
-    serializer.write_output(args.output)
-
-    logger.info("Done")
-
-if __name__ == "__main__":
-    main()

data/.~lock.ringe.xlsx#

@@ -0,0 +1 @@
+,someone,nixLap,30.03.2026 10:01,file:///home/someone/.config/yolk/eggs/libreoffice/4;

data/.~lock.ringe_short.xlsx#

@@ -0,0 +1 @@
+,someone,nixMain,31.03.2026 11:57,file:///home/someone/.config/yolk/eggs/libreoffice/4;

fledermaus_formatter/__main__.py

@@ -0,0 +1,4 @@
+from fledermaus_formatter.main import main
+
+if __name__ == "__main__":
+    main()

fledermaus_formatter/instructions/countings_formater.py

@@ -19,31 +19,40 @@ EXPECTED_OUTPUT_TABLE_SCHEMA.remove("Geschlecht")
 COL_INSTRUCTIONS = {
     ("Art",): {
         "command": "FIND_AND_REPLACE",
-        "map": SPICIES
+        "map": SPICIES,
+        "reqire_full_match": True
     },
-    ("Ort",): {
+    ("Ort",): [
+        {
+            "command": "FIND_AND_REPLACE",
+            "map": CATCHED_BY
+        },
+        {
             "command": "REGEX_MOVE_TO_COL",
-        "with_input_iteration": CATCHED_BY,  # Moves matching group {{% 0 %}} to "to"
-        "match": r"^(?:[^ ]* +)*({{% 0 %}}),?(?: .*+)*$",
-        "to": "Fangart"
-    },
-    ("Anzahl",): {
+            "with_input_iteration": list(CATCHED_BY.keys()),  # Moves matching group {{% 1 %}} to "to"
+            "match": r"^.*({{% 0 %}}),?(?:.*+)*$",
+            "to": "Fangart",
+            "copy": True
+        }
+    ],
+    ("Anzahl",): [{
         "command": "REGEX_EDIT_COL",
         "match": r"^ *$",
-        "replace_with": "1"
+        "replace_with": "1",
+        "match_none": True
         },
-    ("Anzahl",): {
+        {
         "command": "REGEX_GROUP_VALUEBASED_CHOOS",
         "match": r"^ *([0-9]+) *oder *([0-9]+) *$",
         "highest": True
-    },
+        }],
 }
 
 ROW_INSTRUCTION = [
     {
         "command": "CONDITIONAL_COPY",
         "when_regex": {
-            "Geschlecht": r"^(?:[^ ]* +)*(m),?(?: .*+)*$",
+            "Geschlecht": r"^.*(m),?(?:.*+)*$",
             "Anzahl": r"([0-9]+)"
         },
         "source": "Anzahl",
@@ -52,7 +61,7 @@ ROW_INSTRUCTION = [
     {
         "command": "CONDITIONAL_COPY",
         "when_regex": {
-            "Geschlecht": r"^(?:[^ ]* +)*(w),?(?: .*+)*$",
+            "Geschlecht": r"^.*(w),?(?:.*+)*$",
             "Anzahl": r"([0-9]+)"
         },
         "source": "Anzahl",

fledermaus_formatter/instructions/mappings.py

@@ -1,3 +1,17 @@
+def flatten_dict(d):
+    flat = {}
+    for key, values in d.items():
+        for value in values:
+            flat[value] = key
+    return flat
+
+def add_cased_options(inp: str):
+    return set([
+        inp,
+        inp.lower(),
+        inp.upper(),
+        inp.capitalize(),
+    ])
 SPICIES = {
     "Bbar": [
         "B.bar.",
@@ -6,6 +20,7 @@ SPICIES = {
         "bbar",
         "Barbastella",
         "barbastellus",
+        "Bbararar",
         "B.b.",
         "B.b",
         "BB",
@@ -37,6 +52,7 @@ SPICIES = {
         "P.pip.",
         "P.pip",
         "PPIP",
+        "P.p.",
         "ppip",
         "Pipistrellus pipistrellus",
     ],
@@ -155,6 +171,7 @@ SPICIES = {
         "P.nat.",
         "P.nat",
         "PNAT",
+        "Pnatatat",
         "pnat",
         "Pipistrellus nathusii",
         "P.n.",
@@ -177,6 +194,7 @@ SPICIES = {
         "unbekannt",
         "unb.",
         "Unbestimmt",
+        "Mspecbech",
     ],
     "Malc": [
         "M.alc.",
@@ -227,6 +245,7 @@ SPICIES = {
         "MSPEC",
         "mspec",
         "Myotis",
+        "Mspec.Mspec",
         "M.spe.",
         "M.spe",
         "MSPE",
@@ -251,6 +270,7 @@ SPICIES = {
         "MMYO",
         "mmyo",
         "Myotis Myotis",
+        "Mspec.bech",
         "M.m.",
         "M.m",
         "MM",
@@ -292,30 +312,41 @@ SPICIES = {
     ],
 }
 
-CATCHED_BY = [
-    "Nfg",
-    "WQK",
-    "Wst",
-    "KK",
-    "AFZ",
-    "Detektor",
-]
+CATCHED_BY = {
+    "Nfg": [*add_cased_options("nfg"),
+            *add_cased_options("nf")],
+    "KK": [*add_cased_options("kk"), 
+           *add_cased_options(" k "),
+           *add_cased_options("überwinterungskästen"), 
+           *add_cased_options("kastenkontrolle"),
+           *add_cased_options("kasten")],
+    "WQK": [*add_cased_options("wqk"),
+            *add_cased_options("Winterquartierkontrolle"),
+            *add_cased_options("Winterquartierkasten"),
+            *add_cased_options("Winterquartier"),
+            *add_cased_options("kontrolle")],
+    "Wst": [*add_cased_options("wst"),
+            *add_cased_options("Wochenstuben"),
+            *add_cased_options("Wochenstube")],
+    "AFZ": [*add_cased_options("afz")],
+    "Detektor": [*add_cased_options("detektor")],
+}
 
 AGE = {
     "j": [
         "j",
-        "juv"
+        "juv",
     ],
     "sub": [
         "sub",
         "subad",
-        "sub ad"
+        "sub ad",
         "sub adult",
-        "sub-adult"
+        "sub-adult",
     ],
     "a": [
         "a",
-        "ad"
+        "ad",
     ]
 }
 
@@ -323,21 +354,15 @@ REPRODUCTION = {
     "p": [
         "plak",
         "pl",
-        "p"
+        "p",
     ],
     "g": [
-        "g"
+        "g",
     ],
     "z": [
         "lak",
         "l",
-        "z"
+        "z",
     ]
 }
 
-def flatten_dict(d):
-    flat = {}
-    for key, values in d.items():
-        for value in values:
-            flat[value] = key
-    return flat

fledermaus_formatter/instructions/ring_formater.py

@@ -23,30 +23,24 @@ COL_INSTRUCTIONS = {
     ("Geschlecht",): [
     {
         "command": "FIND_AND_REPLACE",
-        "map": REPRODUCTION
+        "map": REPRODUCTION,
     },
     {
         "command": "FIND_AND_REPLACE",
-        "map": AGE
+        "map": AGE,
     },
     {
         "command": "REGEX_MOVE_TO_COL",
         "with_input_iteration": list(REPRODUCTION.keys()),  # Moves matching group {{% 1 %}} to "to"
-        "match": r"^(?:[^ ]* +)*({{% 0 %}}),?(?: .*+)*$",
+        "match": r"^.*({{% 0 %}}),?(?:.*+)*$",
         "to": "Reproduktionsstatus"
     },
      {
         "command": "REGEX_MOVE_TO_COL",
         "with_input_iteration": list(AGE.keys()),  # Moves matching group {{% 1 %}} to "to"
-        "match": r"^(?:[^ ]* +)*({{% 0 %}}),?(?: .*+)*$",
+        "match": r"^.*({{% 0 %}}),?(?:.*+)*$",
         "to": "Alter"
         
-    },
-     {
-        # Special case ma -> Geschlecht a | Alter a
-        "command": "REGEX_MOVE_TO_COL",
-        "match": r"^(?:[^ ]* +)*m(a),?(?: .*+)*$",
-        "to": "Alter"
     }],
     ("Ringnummer",): {
         "command": "REGEX_EDIT_COL",
@@ -56,14 +50,21 @@ COL_INSTRUCTIONS = {
     },
     ("Art",): {
         "command": "FIND_AND_REPLACE",
-        "map": SPICIES
+        "map": SPICIES,
+        "reqire_full_match": True
     },
-    ("Ort",): {
+    ("Ort",): [
+        {
+            "command": "FIND_AND_REPLACE",
+            "map": CATCHED_BY
+        },
+        {
             "command": "REGEX_MOVE_TO_COL",
-        "with_input_iteration": CATCHED_BY,  # Moves matching group {{% 1 %}} to "to"
-        "match": r"^(?:[^ ]* +)*({{% 0 %}}),?(?: .*+)*$",
+            "with_input_iteration": list(CATCHED_BY.keys()),  # Moves matching group {{% 1 %}} to "to"
+            "match": r"^.*({{% 0 %}}),?(?:.*+)*$",
             "to": "Fangart"
         }
+    ]
 }
 
 
@@ -73,11 +74,8 @@ ROW_INSTRUCTIONS = [
         "command": "ROW_COLOR_TO_BOOL",  # Match when found as text color in any col of the row
         "fg_colors": [(255,0,0), (0,0,255)],
         "approximate": True,
-        "threshold": "ALL", # ALL, ANY, MAJORITY of row must be collored
+        "threshold": "ANY", # ALL, ANY, MAJORITY of row must be collored
         "target": "Wiederfunde",
-        "value": "TRUE",
+        "value": "WAHR",
     }
-    #{
-    #    "command": "ROW_COPY_REMAINING"
-    #}
 ]

fledermaus_formatter/logger.py

@@ -0,0 +1,3 @@
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)

fledermaus_formatter/main.py

@@ -0,0 +1,108 @@
+import argparse
+import os
+from fledermaus_formatter.logger import logger 
+
+from fledermaus_formatter.table_handler.serializer import TableSerializer
+import fledermaus_formatter.instructions.countings_formater as countings_formater
+import fledermaus_formatter.instructions.ring_formater as ring_formater
+
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+
+
+
+def process_file(input_file, output_file, xlsx_type):
+    # Select formatter
+    if xlsx_type == "ring":
+        formatter = ring_formater
+        row_instructions = ring_formater.ROW_INSTRUCTIONS
+    else:
+        formatter = countings_formater
+        row_instructions = countings_formater.ROW_INSTRUCTION
+
+    logger.info(f"Processing: {input_file}")
+
+    serializer = TableSerializer(
+        input_file,
+        formatter.EXPECTED_INPUT_TABLE_SCHEMA,
+        formatter.EXPECTED_OUTPUT_TABLE_SCHEMA
+    )
+
+    serializer.run(
+        formatter.COL_INSTRUCTIONS,
+        row_instructions
+    )
+
+    serializer.write_output(output_file)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Serialize Excel tables.")
+
+    # Single file mode (optional fallback)
+    parser.add_argument("-i", "--input", help="Input Excel file")
+    parser.add_argument("-o", "--output", help="Output Excel file")
+    parser.add_argument("-v", help="Version", action="store_true")
+
+    # Directory mode
+    parser.add_argument("--input-dir", help="Input directory")
+    parser.add_argument("--output-dir", help="Output directory")
+    parser.add_argument("-s", help="Serial mode", action="store_true")
+
+    parser.add_argument(
+        "-t", "--type",
+        required=False,
+        choices=["ring", "counting"],
+        help="Type of table to process"
+    )
+
+    args = parser.parse_args()
+
+    if args.v:
+        print("1.5")
+        return
+    if not args.type:
+        parser.error("The --type argument is required.")
+        retur
+
+    # directory
+    if args.input_dir and args.output_dir:
+        logger.info("Running in directory mode")
+
+        tasks = []
+
+        for root, _, files in os.walk(args.input_dir):
+            for file in files:
+                if not file.endswith((".xlsx", ".xls")):
+                    continue
+
+                input_path = os.path.join(root, file)
+
+                relative_path = os.path.relpath(root, args.input_dir)
+                output_root = os.path.join(args.output_dir, relative_path)
+                os.makedirs(output_root, exist_ok=True)
+
+                output_path = os.path.join(output_root, file)
+
+                tasks.append((input_path, output_path, args.type))
+
+        max_workers = os.cpu_count() or 1 if not args.s else 1 
+        logger.info(f"Processing {len(tasks)} files using {max_workers} workers")
+
+        with ProcessPoolExecutor(max_workers=max_workers) as executor:
+            futures = [executor.submit(process_file, *task) for task in tasks]
+
+            for future in as_completed(futures):
+                try:
+                    future.result()
+                except Exception as e:
+                    logger.error(f"Error processing file: {e}")
+        # single file
+    elif args.input and args.output:
+        logger.info("Running in single file mode")
+        process_file(args.input, args.output, args.type)
+
+    else:
+        parser.error("You must provide either input/output files or input-dir/output-dir")
+
+    logger.info("Done")

fledermaus_formatter/table_handler/Entry.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Entry:
+    value: str|int|float|bool|None
+    bg_color: tuple[str, str, str] = (255,255,255)
+    fg_color: tuple[str, str, str] = (255,255,255)

fledermaus_formatter/table_handler/commands/copy.py

@@ -0,0 +1,13 @@
+from fledermaus_formatter.table_handler.Entry import Entry
+
+class Copy:
+    def __init__(self, source: str, target: str, replace = False):
+        self.source = source
+        self.target = target
+        self.replace = replace
+
+    def execute_on_row(self, row=dict[str, str], source_col=None):
+        return {
+            **({} if not self.replace else {self.source_col: Entry(value="")}),
+            self.target: Entry(row[self.source].value, bg_color=row[self.source].bg_color, fg_color=row[self.source].fg_color)
+        }

fledermaus_formatter/table_handler/commands/find_and_replace.py

@@ -0,0 +1,22 @@
+from fledermaus_formatter.table_handler.Entry import Entry
+from copy import deepcopy
+
+class FindAndReplace:
+    def __init__(self, map: dict[str, list[str]], reqire_full_match: bool = False):
+        self.map = map
+        self.require_full_match = reqire_full_match
+
+    def execute_on_cell(self, entry) -> str:
+        res = deepcopy(entry.value)
+
+        for correct_name, bad_names in self.map.items():
+            for bad_name in bad_names:
+                if self.require_full_match and bad_name == res:
+                    res = correct_name
+                    break
+                elif not self.require_full_match:
+                    prior_val = res
+                    res = res.replace(bad_name, correct_name)
+                    if prior_val != res or correct_name in res:
+                        break
+        return Entry(value=res, fg_color=entry.fg_color, bg_color=entry.bg_color)

fledermaus_formatter/table_handler/commands/regex_edit_col.py

@@ -1,10 +1,11 @@
 import regex
 
 class RegexEditCol:
-    def __init__(self, match, replace_with, matching_group_rearange: bool = False):
+    def __init__(self, match, replace_with, matching_group_rearange: bool = False, match_none: bool = False):
         self.match = match
         self.replace_with = replace_with
         self.matching_group_rearange = matching_group_rearange
+        self.match_none = match_none
 
     def _reorder(self, value: str):
         matches = regex.match(self.match, value)
@@ -18,9 +19,12 @@ class RegexEditCol:
     def execute_on_cell(self, entry) -> str:
         value = entry.value 
         if value is None:
-            return value
+            if self.match_none:
+                entry.value = self.replace_with
+            return entry
         
         if self.matching_group_rearange:
-            return self._reorder(value)
-
-        return regex.sub(self.match, self.replace_with, value)
+            entry.value = self._reorder(value)
+        else:
+            entry.value = regex.sub(self.match, self.replace_with, value)
+        return entry

fledermaus_formatter/table_handler/commands/regex_group_valuebased_choose.py

@@ -15,7 +15,8 @@ class RegexGroupValuebasedChoose:
         if not len(values):
             return None
         if self.highest:
-            return str(values[-1])
+            entry.value = str(values[-1])
         else:
-            return str(values[0])
+            entry.value = str(values[0])
+        return entry
         

fledermaus_formatter/table_handler/commands/regex_move_to_col.py

@@ -1,10 +1,12 @@
 import regex
 import re
+from fledermaus_formatter.table_handler.Entry import Entry
 
 class RegexMoveToCol:
-    def __init__(self, match: str, to: str, with_input_iteration: list[str]|None = None):
+    def __init__(self, match: str, to: str, with_input_iteration: list[list[str]]|None = None, copy: bool = False):
         self.match = match
         self.to = to
+        self.copy = copy
 
         if with_input_iteration is not None:
             with_input_iteration = (
@@ -52,25 +54,28 @@ class RegexMoveToCol:
         return regex.sub(repl, value)
 
     def do_changes(self, source_coll: str, used_regex: str, row: dict[str, str], changes={}):
-        current_to = changes.get(self.to, row[source_coll].value)
-        current_source = changes.get(source_coll, row[source_coll].value)
+        current_to = changes.get(self.to, row[source_coll]).value
+        current_source = changes.get(source_coll, row[source_coll]).value
         
         pending_res_to = self._get_merged_groups(used_regex, current_to)
         if pending_res_to is not None:
-            changes[self.to] = pending_res_to 
-            changes[source_coll] = self._remove_regex_groups(used_regex, current_source)
+            changes[self.to] = Entry(value=pending_res_to, fg_color=row[self.to].fg_color, bg_color=row[self.to].bg_color)
+            changes[source_coll] = Entry(value=self._remove_regex_groups(used_regex, current_source), fg_color=row[source_coll].fg_color, bg_color=row[source_coll].bg_color)
         return changes  
 
     def execute_on_row(self, row: dict[str, str], source_coll: str = None):
         if not source_coll:
             raise ValueError("source_col required")
-        if self.with_input_iteration:
         changes = {}
+        
+        if self.with_input_iteration:
             for iteration in self.with_input_iteration:
                 used_regex = self.match
                 for operation_id in range(0, self.__operation_input_depth):
                     used_regex = used_regex.replace(f"{{{{% {operation_id} %}}}}", iteration[operation_id])
                 changes = self.do_changes(source_coll, used_regex, row, changes)
-            return changes
         else:
-            return self.do_changes(source_coll, self.match, row)
+            changes = self.do_changes(source_coll, self.match, row)
+        if self.copy and source_coll in changes:
+            del changes[source_coll]
+        return changes

fledermaus_formatter/table_handler/commands/row_color_to_value.py

@@ -1,3 +1,12 @@
+from fledermaus_formatter.table_handler.Entry import Entry
+from math import sqrt
+
+def div_from_color(given_rgb, reqiuired_rgb):
+    r, g, b = given_rgb
+    cr, cg, cb = reqiuired_rgb
+    color_diff = sqrt((r - cr)**2 + (g - cg)**2 + (b - cb)**2)
+    return color_diff
+
 class RowColorToCalue:
     def __init__(self, target: str,
                 value: str,
@@ -13,9 +22,9 @@ class RowColorToCalue:
         self.fg_color = fg_colors if fg_colors else []
         self.bg_color = bg_colors if bg_colors else []
 
-        self.value = value,
-        self.target = target,
-        self.approximate = approximate,
+        self.value = value
+        self.target = target
+        self.approximate = approximate
 
         if threshold not in ("ALL", "ANY", "MAJORITY"):
             raise ValueError("threshold must be ALL, ANY or MAJORITY")
@@ -23,12 +32,16 @@ class RowColorToCalue:
 
       
     def _check_if_collection_matchers(self, collectin: dict[tuple[int, int, int]], color_place: str) -> bool:
+        threshold_sq = 80
+
         res_map = []
         if self.approximate:
-            margin = 10
             for color in collectin:
-                for allowed_color in self.__dict__[color_place]:
-                    res_map.append(all(abs(a - b) <= margin for a, b in zip(color, allowed_color)))
+                is_match = any(
+                    div_from_color(color, allowed_color) <= threshold_sq
+                    for allowed_color in self.__dict__[color_place]
+                )
+                res_map.append(is_match)
         else:
             res_map = list(map(lambda x: x in self.__dict__[color_place]))
 
@@ -44,9 +57,9 @@ class RowColorToCalue:
     def execute_on_row(self, row: dict[str, str], source_coll: str|None = None):
         color_pace_results = []
         for color_place in ["fg_color", "bg_color"]:
-            color_collection = (map(lambda c: c.__dict__[color_place], row.values()))
+            color_collection = (map(lambda c: c.__dict__[color_place], filter(lambda x: x.value, row.values())))
             color_pace_results.append(self._check_if_collection_matchers(color_collection, color_place))
         
-        if all(color_pace_results):
-            return {self.target: self.value}
+        if any(color_pace_results):
+            return {self.target: Entry(value=self.value, fg_color=row[self.target].fg_color, bg_color=row[self.target].bg_color)}
         return {}

fledermaus_formatter/table_handler/serializer.py

@@ -1,10 +1,9 @@
 
 from openpyxl import Workbook, load_workbook
-from dataclasses import dataclass
 from pprint import pprint as print
 
-from table_handler.utils import hex_to_rgb, delegate_tasks
-from logger import lock_logger
+import fledermaus_formatter.instructions.mappings as mappings
+from fledermaus_formatter.table_handler.Entry import Entry
 
 from .commands.find_and_replace import FindAndReplace
 from .commands.regex_edit_col import RegexEditCol
@@ -12,27 +11,34 @@ from .commands.regex_move_to_col import RegexMoveToCol
 from .commands.copy import Copy
 from .commands.regex_group_valuebased_choose import RegexGroupValuebasedChoose
 from .commands.row_color_to_value import RowColorToCalue
+from fledermaus_formatter.logger import logger
 
 from .conditions.regex_cell import RegexCellCondition
 
-from multiprocessing import Pool, Queue, Process, Manager, cpu_count 
+def hex_to_rgb(hex):
+    if type(hex) != str and str("Values must be of type <class 'str'>"):
+        # chatch wierd openxl things
+        hex = "#0"
+    hex=  "{:06}".format(str(hex).lstrip("#"))
+    return tuple(int(hex[i:i+2],16) for i in (0, 2, 4))
 
-@dataclass
-class Entry:
-    value: str|int|float|bool|None
-    bg_color: tuple[str, str, str]
-    fg_color: tuple[str, str, str]
-
-
-
-class XslxReader:
-    def __init__(self, file):
+class TableSerializer:
+    def __init__(self, file: str, expected_input_schema: list[str], expected_output_schema: list[str]):
         self.wb: Workbook = load_workbook(file, read_only=True)
         self.ws = self.wb.active
         if not self.ws:
             raise ValueError("No active worksheet found in the workbook.")
         
-    def serialize_table(self, expected_cols: list[str]) -> dict[int, list[Entry]]:
+        self.file = file
+        self.data = self._serialize_table(expected_input_schema)
+        self.serialized_output = {}
+        self.seperator = ", "
+        self.expected_output_schema = expected_output_schema or expected_input_schema
+        self.implicit_copy = True
+        self.output_header = True
+
+
+    def _serialize_table(self, expected_cols: list[str]) -> dict[int, list[Entry]]:
         data = {col: {} for col in expected_cols}
         map_expected_with_ws = {}
         for idx, cell in enumerate(self.ws[1]):
@@ -49,25 +55,14 @@ class XslxReader:
             for expected_col, ws_idx in map_expected_with_ws.items():
                 cell = row[ws_idx]
                 data[expected_col][rid] = Entry(
-                    value=str(cell.value) if not cell.value is None else "",
-                    bg_color=hex_to_rgb(cell.fill.start_color.rgb if cell.fill and cell.fill.start_color else "#0"),
-                    fg_color=hex_to_rgb(cell.font.color.rgb if cell.font and cell.font.color else "#0")
+                    value=str(cell._value) if not cell.value is None else "",
+                    bg_color=hex_to_rgb(cell.fill.end_color.rgb[2:] if cell.fill and cell.fill.start_color else "#0"),
+                    fg_color=hex_to_rgb(cell.font.color.rgb[2:] if cell.font and cell.font.color and type(cell.font.color.rgb) == str else "#0")
                 )
         return data
     
-class Transformer:
-    def __init__(self, data, expected_output_schema: list[str], cumulated_instructions = {}):
-        self.data = data
-        self.fully_serialized_output = {}
-        self.seperator = ", "
-        self.expected_output_schema = expected_output_schema
-        self.implicit_copy = True
-        self.output_header = True
-        self.cumulated_instructions = cumulated_instructions
-
-    
     def write_output(self, destination_file: str):
-        if not self.fully_serialized_output:
+        if not self.serialized_output:
             raise ValueError("There is nothing to write, serialize first")
         
         # write header
@@ -79,9 +74,9 @@ class Transformer:
                 ows.cell(row=1, column=col_i, value=col_name)
 
         # write rows
-        for row_i, (row_id, row_data) in enumerate(self.fully_serialized_output.items(), start=2 if self.output_header else 1):
+        for row_i, (row_id, row_data) in enumerate(self.serialized_output.items(), start=2 if self.output_header else 1):
             for col_i, col_name in enumerate(self.expected_output_schema, start=1):
-                ows.cell(row=row_i, column=col_i, value=row_data.get(col_name))
+                ows.cell(row=row_i, column=col_i, value=row_data.get(col_name).value if row_data.get(col_name) else "")
         
         owb.save(destination_file)
 
@@ -101,27 +96,29 @@ class Transformer:
         return value
 
     def _extecute_instruct(self, source_cell: str, instruct: dict, row: dict[str, any], working_row: dict[str, any]):
+        in_place_working_row = {**row, **working_row}
+        
         # execute changes
         res = None
         res_row = {}
         match instruct.get("command"):
             case "FIND_AND_REPLACE":
-                res = FindAndReplace(instruct["map"]).execute_on_cell(row[source_cell])
+                res = FindAndReplace(instruct["map"], instruct.get("reqire_full_match", False)).execute_on_cell(in_place_working_row[source_cell])
             case "REGEX_MOVE_TO_COL":
-                res_row = RegexMoveToCol(instruct["match"], instruct["to"], instruct.get("with_input_iteration")) \
-                    .execute_on_row(row, source_cell)
+                res_row = RegexMoveToCol(instruct["match"], instruct["to"], instruct.get("with_input_iteration"), instruct.get("copy", False)) \
+                    .execute_on_row(in_place_working_row, source_cell)
             case "REGEX_EDIT_COL":
-                res = RegexEditCol(instruct["match"], instruct["replace_with"], instruct.get("matching_group_rearange"))\
-                    .execute_on_cell(row[source_cell])
+                res = RegexEditCol(instruct["match"], instruct["replace_with"], instruct.get("matching_group_rearange"), instruct.get("match_none"))\
+                    .execute_on_cell(in_place_working_row[source_cell])
             case "REGEX_GROUP_VALUEBASED_CHOOS":
                 res = RegexGroupValuebasedChoose(instruct["match"], instruct["highest"])\
-                    .execute_on_cell(row[source_cell])
+                    .execute_on_cell(in_place_working_row[source_cell])
             case "CONDITIONAL_COPY":
                 res_row = Copy(instruct["source"], instruct["target"], replace=instruct.get("replace"))\
-                    .execute_on_row(row)
+                    .execute_on_row(in_place_working_row)
             case "ROW_COLOR_TO_BOOL":
                 res_row = RowColorToCalue(instruct["target"], instruct["value"], instruct.get("fg_colors"), instruct.get("bg_colors"), instruct.get("approximate"), instruct.get("threshold"))\
-                    .execute_on_row(row)
+                    .execute_on_row(in_place_working_row)
 
         # Merge row_changes to working_row
         for target_cel, change in res_row.items():
@@ -132,28 +129,6 @@ class Transformer:
             working_row[source_cell] = self._merge_change_to_cel(working_row.get(source_cell), res)
         return working_row
 
-    def _run_row_resolution(self, range, worker: str = "X", q = None):
-        self.fully_serialized_output = {}
-        end = list(range)[-1]
-        for rid in range:
-            if rid % 100 == 0 and q:
-                q.put(f"Worker {worker}:Working on row {rid}/{end}")
-            working_row = {}
-            source_row = self.get_row_as_dict(rid)
-            for source_cols, instructions in self.cumulated_instructions:
-                for instruct in instructions if type(instructions) == list else [instructions]:
-                    for col in source_cols:
-                        if not self._check_conditions(instruct, source_row):
-                            continue
-                        working_row = self._extecute_instruct(col, instruct, source_row, working_row)
-            if not self.implicit_copy:
-                continue
-            for col in self.    expected_output_schema:
-                if col in working_row:
-                    continue
-                working_row[col] = source_row.get(col).value
-            self.fully_serialized_output[rid] = working_row
-        return self.fully_serialized_output
 
     @property
     def vertical_len(self):
@@ -163,27 +138,25 @@ class Transformer:
         return {col: rows.get(rid) for col, rows in self.data.items()}
 
     def run(self, col_instructions: dict[tuple[str], dict], row_instructions: dict[tuple[str], dict]):
-        self.cumulated_instructions = list(col_instructions.items()) + [
+        self.serialized_output = {}
+        amount_working_rows = self.vertical_len +2
+        for rid in range(2, amount_working_rows):
+            if rid % 1000 == 0:
+                logger.info(f"{self.file} Working on row {rid}/{amount_working_rows} ")
+            working_row = {}
+            source_row = self.get_row_as_dict(rid)
+            for source_cols, instructions in list(col_instructions.items()) + [
                     ((None,), instr) for instr in row_instructions
-                ]
-        amount_working_rows = self.vertical_len + 2
-
-        with Manager() as manager:
-            q = manager.Queue()
-
-            chnucs = [[elem, worker_name, q] for worker_name, elem in enumerate(
-                delegate_tasks(range(2, amount_working_rows), cpu_count(), 1000)
-            )]
-
-            log_proc = Process(target=lock_logger, args=(q,))
-            log_proc.start()
-
-            with Pool(len(chnucs)) as p:
-                result = p.starmap(
-                    Transformer(self.data, self.expected_output_schema, self.cumulated_instructions)._run_row_resolution,
-                    chnucs
-                )
-            q.put("DONE")
-            log_proc.join(1)
-        self.fully_serialized_output = {k: v for d in result for k, v in d.items()}
-        return
+                ]:
+                for instruct in instructions if type(instructions) == list else [instructions]:
+                    for col in source_cols:
+                        if not self._check_conditions(instruct, source_row):
+                            continue
+                        working_row = self._extecute_instruct(col, instruct, source_row, working_row)
+            if not self.implicit_copy:
+                continue
+            for col in self.expected_output_schema:
+                if col in working_row:
+                    continue
+                working_row[col] = source_row.get(col)
+            self.serialized_output[rid] = working_row

install.sh

@@ -0,0 +1,22 @@
+#!/usr/share/env bash
+
+# Target dir is config vecause this will never include binary blops
+TARGET_DIR="$HOME/.config/fledermaus-formatter"
+if [ -d $TARGET_DIR ]; then
+    rm -rf "${TARGET_DIR}"
+fi
+mkdir -p $TARGET_DIR
+git clone git@git.urmel.de:Kyattsukuro/fledermaus.git "$TARGET_DIR"
+python3 -m venv $TARGET_DIR/.venv
+source $TARGET_DIR/.venv/bin/activate
+pip install -r $TARGET_DIR/requirements.txt
+
+mkdir -p $HOME/.local/share/bin/
+
+rm -r $HOME/.local/share/bin/fledermaus-formater > /dev/zero
+ln -s $TARGET_DIR/run.sh $HOME/.local/share/bin/fledermaus-formater
+
+if [[ ":$PATH:" != *":$HOME/.local/share/bin:"* ]]; then
+    echo 'export PATH="$HOME/.local/share/bin:$PATH"' >> ~/.bashrc
+    export PATH="$HOME/.local/share/bin:$PATH"
+fi

logger.py

@@ -1,10 +0,0 @@
-import logging
-logger = logging.getLogger(__name__)
-logging.basicConfig(level=logging.INFO)
-
-def lock_logger(q):
-    while True:
-        msg = q.get()
-        if msg == "DONE":
-            break
-        logger.info(msg)

run.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR="$HOME/.config/fledermaus-formatter"
+
+source $SCRIPT_DIR/.venv/bin/activate
+export PYTHONPATH=$SCRIPT_DIR:$PYTHONPATH
+$SCRIPT_DIR/.venv/bin/python3 -m fledermaus_formatter $@

table_handler/commands/copy.py

@@ -1,13 +0,0 @@
-
-
-class Copy:
-    def __init__(self, source: str, target: str, replace = False):
-        self.source = source
-        self.target = target
-        self.replace = replace
-
-    def execute_on_row(self, row=dict[str, str], source_col=None):
-        return {
-            **({} if not self.replace else {self.source_col: ""}),
-            self.target: row[self.source].value
-        }

table_handler/commands/find_and_replace.py

@@ -1,20 +0,0 @@
-import re
-from copy import deepcopy
-
-class FindAndReplace:
-    def __init__(self, map: dict[str, list[str]], case_sensitive: bool = False):
-        self.map = map
-        self.case_sensitive = case_sensitive
-
-    def execute_on_cell(self, entry) -> str:
-        res = deepcopy(entry.value)
-
-        for species, alternatives in self.map.items():
-            for old_version in alternatives:
-                if self.case_sensitive:
-                    res = res.replace(old_version, species)
-                else:
-                    pattern = re.compile(re.escape(old_version), re.IGNORECASE)
-                    res = pattern.sub(species, res)
-
-        return res

table_handler/utils.py

@@ -1,20 +0,0 @@
-def hex_to_rgb(hex):
-    if type(hex) != str and str("Values must be of type <class 'str'>"):
-        # chatch wierd openxl things
-        hex = "#0"
-    hex=  "{:06}".format(str(hex).lstrip("#"))
-    return tuple(int(hex[i:i+2],16) for i in (0, 2, 4))
-
-def delegate_tasks(tasks, threats, min_tasks_per_threat):
-    n = len(tasks)
-
-    # determine usable cores
-    usable_cores = min(threats, n // min_tasks_per_threat)
-
-    if usable_cores == 0:
-        usable_cores = 1
-
-    # split tasks evenly
-    chunk_size = (n + usable_cores - 1) // usable_cores
-    chunks = [tasks[i:i + chunk_size] for i in range(0, n, chunk_size)]
-    return chunks