Compare commits
No commits in common. "a4758e42f3a05c36a5b991c833b31c2ac997773d" and "21a715149eb5b2e0eb116e3e3517a7b43d763694" have entirely different histories.
a4758e42f3
...
21a715149e
110
__init__.py
110
__init__.py
@ -1,51 +1,26 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import os
|
from table_handler.serializer import TableSerializer
|
||||||
from logger import logger
|
from logger import logger
|
||||||
|
|
||||||
from table_handler.serializer import TableSerializer
|
|
||||||
import instructions.countings_formater as countings_formater
|
import instructions.countings_formater as countings_formater
|
||||||
import instructions.ring_formater as ring_formater
|
import instructions.ring_formater as ring_formater
|
||||||
|
|
||||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def process_file(input_file, output_file, xlsx_type):
|
|
||||||
# Select formatter
|
|
||||||
if xlsx_type == "ring":
|
|
||||||
formatter = ring_formater
|
|
||||||
row_instructions = ring_formater.ROW_INSTRUCTIONS
|
|
||||||
else:
|
|
||||||
formatter = countings_formater
|
|
||||||
row_instructions = countings_formater.ROW_INSTRUCTION
|
|
||||||
|
|
||||||
logger.info(f"Processing: {input_file}")
|
|
||||||
|
|
||||||
serializer = TableSerializer(
|
|
||||||
input_file,
|
|
||||||
formatter.EXPECTED_INPUT_TABLE_SCHEMA,
|
|
||||||
formatter.EXPECTED_OUTPUT_TABLE_SCHEMA
|
|
||||||
)
|
|
||||||
|
|
||||||
serializer.run(
|
|
||||||
formatter.COL_INSTRUCTIONS,
|
|
||||||
row_instructions
|
|
||||||
)
|
|
||||||
|
|
||||||
serializer.write_output(output_file)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description="Serialize Excel tables.")
|
parser = argparse.ArgumentParser(description="Serialize Excel tables.")
|
||||||
|
|
||||||
# Single file mode (optional fallback)
|
parser.add_argument(
|
||||||
parser.add_argument("-i", "--input", help="Input Excel file")
|
"-i", "--input",
|
||||||
parser.add_argument("-o", "--output", help="Output Excel file")
|
required=True,
|
||||||
|
help="Input Excel file"
|
||||||
|
)
|
||||||
|
|
||||||
# Directory mode
|
parser.add_argument(
|
||||||
parser.add_argument("--input-dir", help="Input directory")
|
"-o", "--output",
|
||||||
parser.add_argument("--output-dir", help="Output directory")
|
required=True,
|
||||||
|
help="Output Excel file"
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-t", "--type",
|
"-t", "--type",
|
||||||
@ -56,48 +31,31 @@ def main():
|
|||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# directory
|
# Select formatter based on type
|
||||||
if args.input_dir and args.output_dir:
|
if args.type == "ring":
|
||||||
logger.info("Running in directory mode")
|
formatter = ring_formater
|
||||||
|
row_instructions = ring_formater.ROW_INSTRUCTIONS
|
||||||
tasks = []
|
|
||||||
|
|
||||||
for root, _, files in os.walk(args.input_dir):
|
|
||||||
for file in files:
|
|
||||||
if not file.endswith((".xlsx", ".xls")):
|
|
||||||
continue
|
|
||||||
|
|
||||||
input_path = os.path.join(root, file)
|
|
||||||
|
|
||||||
relative_path = os.path.relpath(root, args.input_dir)
|
|
||||||
output_root = os.path.join(args.output_dir, relative_path)
|
|
||||||
os.makedirs(output_root, exist_ok=True)
|
|
||||||
|
|
||||||
output_path = os.path.join(output_root, file)
|
|
||||||
|
|
||||||
tasks.append((input_path, output_path, args.type))
|
|
||||||
|
|
||||||
max_workers = os.cpu_count() or 1
|
|
||||||
logger.info(f"Processing {len(tasks)} files using {max_workers} workers")
|
|
||||||
|
|
||||||
with ProcessPoolExecutor(max_workers=max_workers) as executor:
|
|
||||||
futures = [executor.submit(process_file, *task) for task in tasks]
|
|
||||||
|
|
||||||
for future in as_completed(futures):
|
|
||||||
try:
|
|
||||||
future.result()
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error processing file: {e}")
|
|
||||||
# single file
|
|
||||||
elif args.input and args.output:
|
|
||||||
logger.info("Running in single file mode")
|
|
||||||
process_file(args.input, args.output, args.type)
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
parser.error("You must provide either input/output files or input-dir/output-dir")
|
formatter = countings_formater
|
||||||
|
row_instructions = countings_formater.ROW_INSTRUCTION
|
||||||
|
|
||||||
|
logger.info("Reading file, building serializer...")
|
||||||
|
serializer = TableSerializer(
|
||||||
|
args.input,
|
||||||
|
formatter.EXPECTED_INPUT_TABLE_SCHEMA,
|
||||||
|
formatter.EXPECTED_OUTPUT_TABLE_SCHEMA
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Starting transformation")
|
||||||
|
serializer.run(
|
||||||
|
formatter.COL_INSTRUCTIONS,
|
||||||
|
row_instructions
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.info("Writing transformations")
|
||||||
|
serializer.write_output(args.output)
|
||||||
|
|
||||||
logger.info("Done")
|
logger.info("Done")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
@ -6,7 +6,6 @@ SPICIES = {
|
|||||||
"bbar",
|
"bbar",
|
||||||
"Barbastella",
|
"Barbastella",
|
||||||
"barbastellus",
|
"barbastellus",
|
||||||
"Bbararar",
|
|
||||||
"B.b.",
|
"B.b.",
|
||||||
"B.b",
|
"B.b",
|
||||||
"BB",
|
"BB",
|
||||||
@ -38,7 +37,6 @@ SPICIES = {
|
|||||||
"P.pip.",
|
"P.pip.",
|
||||||
"P.pip",
|
"P.pip",
|
||||||
"PPIP",
|
"PPIP",
|
||||||
"P.p."
|
|
||||||
"ppip",
|
"ppip",
|
||||||
"Pipistrellus pipistrellus",
|
"Pipistrellus pipistrellus",
|
||||||
],
|
],
|
||||||
@ -157,7 +155,6 @@ SPICIES = {
|
|||||||
"P.nat.",
|
"P.nat.",
|
||||||
"P.nat",
|
"P.nat",
|
||||||
"PNAT",
|
"PNAT",
|
||||||
"Pnatatat",
|
|
||||||
"pnat",
|
"pnat",
|
||||||
"Pipistrellus nathusii",
|
"Pipistrellus nathusii",
|
||||||
"P.n.",
|
"P.n.",
|
||||||
@ -180,7 +177,6 @@ SPICIES = {
|
|||||||
"unbekannt",
|
"unbekannt",
|
||||||
"unb.",
|
"unb.",
|
||||||
"Unbestimmt",
|
"Unbestimmt",
|
||||||
"Mspecbech"
|
|
||||||
],
|
],
|
||||||
"Malc": [
|
"Malc": [
|
||||||
"M.alc.",
|
"M.alc.",
|
||||||
@ -231,7 +227,6 @@ SPICIES = {
|
|||||||
"MSPEC",
|
"MSPEC",
|
||||||
"mspec",
|
"mspec",
|
||||||
"Myotis",
|
"Myotis",
|
||||||
"Mspec.Mspec",
|
|
||||||
"M.spe.",
|
"M.spe.",
|
||||||
"M.spe",
|
"M.spe",
|
||||||
"MSPE",
|
"MSPE",
|
||||||
@ -256,7 +251,6 @@ SPICIES = {
|
|||||||
"MMYO",
|
"MMYO",
|
||||||
"mmyo",
|
"mmyo",
|
||||||
"Myotis Myotis",
|
"Myotis Myotis",
|
||||||
"Mspec.bech"
|
|
||||||
"M.m.",
|
"M.m.",
|
||||||
"M.m",
|
"M.m",
|
||||||
"MM",
|
"MM",
|
||||||
|
|||||||
@ -77,4 +77,7 @@ ROW_INSTRUCTIONS = [
|
|||||||
"target": "Wiederfunde",
|
"target": "Wiederfunde",
|
||||||
"value": "TRUE",
|
"value": "TRUE",
|
||||||
}
|
}
|
||||||
|
#{
|
||||||
|
# "command": "ROW_COPY_REMAINING"
|
||||||
|
#}
|
||||||
]
|
]
|
||||||
@ -2,13 +2,19 @@ import re
|
|||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
|
|
||||||
class FindAndReplace:
|
class FindAndReplace:
|
||||||
def __init__(self, map: dict[str, list[str]]):
|
def __init__(self, map: dict[str, list[str]], case_sensitive: bool = False):
|
||||||
self.map = map
|
self.map = map
|
||||||
|
self.case_sensitive = case_sensitive
|
||||||
|
|
||||||
def execute_on_cell(self, entry) -> str:
|
def execute_on_cell(self, entry) -> str:
|
||||||
res = deepcopy(entry.value)
|
res = deepcopy(entry.value)
|
||||||
|
|
||||||
for species, alternatives in self.map.items():
|
for species, alternatives in self.map.items():
|
||||||
for old_version in alternatives:
|
for old_version in alternatives:
|
||||||
res = res.replace(old_version, species)
|
if self.case_sensitive:
|
||||||
|
res = res.replace(old_version, species)
|
||||||
|
else:
|
||||||
|
pattern = re.compile(re.escape(old_version), re.IGNORECASE)
|
||||||
|
res = pattern.sub(species, res)
|
||||||
|
|
||||||
return res
|
return res
|
||||||
@ -37,7 +37,6 @@ class TableSerializer:
|
|||||||
if not self.ws:
|
if not self.ws:
|
||||||
raise ValueError("No active worksheet found in the workbook.")
|
raise ValueError("No active worksheet found in the workbook.")
|
||||||
|
|
||||||
self.file = file
|
|
||||||
self.data = self._serialize_table(expected_input_schema)
|
self.data = self._serialize_table(expected_input_schema)
|
||||||
self.serialized_output = {}
|
self.serialized_output = {}
|
||||||
self.seperator = ", "
|
self.seperator = ", "
|
||||||
@ -148,7 +147,7 @@ class TableSerializer:
|
|||||||
amount_working_rows = self.vertical_len +2
|
amount_working_rows = self.vertical_len +2
|
||||||
for rid in range(2, amount_working_rows):
|
for rid in range(2, amount_working_rows):
|
||||||
if rid % 1000 == 0:
|
if rid % 1000 == 0:
|
||||||
logger.info(f"{self.file} Working on row {rid}/{amount_working_rows} ")
|
logger.info(f"Working on row {rid}/{amount_working_rows} ")
|
||||||
working_row = {}
|
working_row = {}
|
||||||
source_row = self.get_row_as_dict(rid)
|
source_row = self.get_row_as_dict(rid)
|
||||||
for source_cols, instructions in list(col_instructions.items()) + [
|
for source_cols, instructions in list(col_instructions.items()) + [
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user