Logging multiprocessing: configure logger dans les workers spawn + compteur de progression
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@ -35,6 +35,15 @@ def setup_logging(verbose=False, debug=False):
|
|||||||
logger.setLevel(level)
|
logger.setLevel(level)
|
||||||
logger.handlers.clear()
|
logger.handlers.clear()
|
||||||
logger.addHandler(handler)
|
logger.addHandler(handler)
|
||||||
|
|
||||||
|
# Also configure the root logger so worker processes log properly
|
||||||
|
root_logger = logging.getLogger()
|
||||||
|
root_logger.setLevel(level)
|
||||||
|
if not root_logger.handlers:
|
||||||
|
root_handler = logging.StreamHandler(sys.stdout)
|
||||||
|
root_handler.setFormatter(logging.Formatter(fmt, datefmt="%H:%M:%S"))
|
||||||
|
root_logger.addHandler(root_handler)
|
||||||
|
|
||||||
return logger
|
return logger
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -261,15 +261,17 @@ class LidarArchaeoPipeline:
|
|||||||
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), self.resolution, self.force): laz_file
|
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), self.resolution, self.force): laz_file
|
||||||
for laz_file in files
|
for laz_file in files
|
||||||
}
|
}
|
||||||
for idx, future in enumerate(as_completed(future_to_file), 1):
|
done = 0
|
||||||
|
for future in as_completed(future_to_file):
|
||||||
laz_file = future_to_file[future]
|
laz_file = future_to_file[future]
|
||||||
|
done += 1
|
||||||
try:
|
try:
|
||||||
success = future.result()
|
success = future.result()
|
||||||
results[laz_file.name] = success
|
results[laz_file.name] = success
|
||||||
status = "✓" if success else "✗"
|
status = "✓" if success else "✗"
|
||||||
logger.info(f" [{idx}/{len(files)}] {status} {laz_file.name}")
|
logger.info(f" [{done}/{len(files)}] {status} {laz_file.name}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f" [{idx}/{len(files)}] ✗ {laz_file.name}: {e}")
|
logger.error(f" [{done}/{len(files)}] ✗ {laz_file.name}: {e}")
|
||||||
logger.debug(f" Traceback:", exc_info=True)
|
logger.debug(f" Traceback:", exc_info=True)
|
||||||
results[laz_file.name] = False
|
results[laz_file.name] = False
|
||||||
else:
|
else:
|
||||||
@ -319,6 +321,16 @@ def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, fo
|
|||||||
|
|
||||||
Each worker gets its own temp directory to avoid file conflicts.
|
Each worker gets its own temp directory to avoid file conflicts.
|
||||||
"""
|
"""
|
||||||
|
# Configure logging in worker process (spawn doesn't inherit parent config)
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
logger = logging.getLogger("lidar")
|
||||||
|
if not logger.handlers:
|
||||||
|
handler = logging.StreamHandler(sys.stdout)
|
||||||
|
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.addHandler(handler)
|
||||||
|
|
||||||
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force)
|
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force)
|
||||||
basename = Path(laz_file_str).stem
|
basename = Path(laz_file_str).stem
|
||||||
pipeline.temp_dir = pipeline.output_dir / f"temp_{basename}"
|
pipeline.temp_dir = pipeline.output_dir / f"temp_{basename}"
|
||||||
|
|||||||
Reference in New Issue
Block a user