OiO.lk Community platform!

Oio.lk is an excellent forum for developers, providing a wide range of resources, discussions, and support for those in the developer community. Join oio.lk today to connect with like-minded professionals, share insights, and stay updated on the latest trends and technologies in the development field.
  You need to log in or register to access the solved answers to this problem.
  • You have reached the maximum number of guest views allowed
  • Please register below to remove this limitation

Logger setup for subprocess logging

  • Thread starter Thread starter Adam
  • Start date Start date
A

Adam

Guest
I'm trying to run a simple python script that writes data from a subprocess to my log file using my class logger.

I've got the following right now, which does works:

Code:
import logging
from itertools import repeat

from multiprocessing import Process, Pool, Queue, cpu_count
from multiprocessing.managers import BaseManager, NamespaceProxy


class SharedObj:
    def __init__(self):
        self.attr1 = 0

class SharedProxy(NamespaceProxy):
    _exposed_ = ("__getattribute__", "__setattr__", "__delattr__")


class QueueManager(BaseManager):
    BaseManager.register("Queue", Queue)
    BaseManager.register("SharedObj", SharedObj, SharedProxy)

class Main:
    def __init__(self):
        self.logger = logging.getLogger(__name__)

    def _loggingSubprocess(self, queue):
        while True:
            try:
                logData = queue.get()
                if logData is None:
                    break
            except EOFError:
                break
            queuedLogger = getattr(self.logger, logData[1])

            queuedLogger(logData[0])

    def _subProc(self, iterNum, sharedObj, queue):
        queue.put((f"In subproc {iterNum}", "debug"))
        sharedObj.attr1 += iterNum

    def run(self):
        with QueueManager() as manager:
            queue = manager.Queue()
            shObj = manager.SharedObj()
            _ = Process(target=self._loggingSubprocess, args=(queue,)).start()
            

            pool = Pool(processes=cpu_count() - 2)
            pool.starmap(
                self._subProc,
                zip(
                    range(10),
                    repeat(shObj),
                    repeat(queue),
                )
            )



def setupLogger():
    logConfig = {   
        "format": "%(asctime)s %(filename)s->%(funcName)s():%(lineno)s %(levelname)s: %(message)s",
        "datefmt": "%m/%d/%Y %I:%M:%S",
        "encoding": "utf-8",
        "loglevel": "DEBUG",
        "warnings": True
    }

    logLevels = {
        "CRITICAL": logging.CRITICAL,
        "DEBUG": logging.DEBUG,
        "ERROR": logging.ERROR,
        "INFO": logging.INFO,
        "WARNING": logging.WARNING,
    }

    logging.basicConfig(
            filename="logs/logFile.log",
            format=logConfig["format"],
            datefmt=logConfig["datefmt"],
            encoding=logConfig["encoding"],
            level=logLevels[logConfig["loglevel"]],
        )

    logging.captureWarnings(logConfig["warnings"])
    
setupLogger()
if __name__ == "__main__":
    
    Main().run()

but if I move the setupLogger() call to inside the __main__ namescape it no longer works correctly.

Code:
if __name__ == "__main__":
    setupLogger()
    Main().run()

Why can't the setupLogger call happen inside __main__?
<p>I'm trying to run a simple python script that writes data from a subprocess to my log file using my class logger.</p>
<p>I've got the following right now, which does works:</p>
<pre><code>import logging
from itertools import repeat

from multiprocessing import Process, Pool, Queue, cpu_count
from multiprocessing.managers import BaseManager, NamespaceProxy


class SharedObj:
def __init__(self):
self.attr1 = 0

class SharedProxy(NamespaceProxy):
_exposed_ = ("__getattribute__", "__setattr__", "__delattr__")


class QueueManager(BaseManager):
BaseManager.register("Queue", Queue)
BaseManager.register("SharedObj", SharedObj, SharedProxy)

class Main:
def __init__(self):
self.logger = logging.getLogger(__name__)

def _loggingSubprocess(self, queue):
while True:
try:
logData = queue.get()
if logData is None:
break
except EOFError:
break
queuedLogger = getattr(self.logger, logData[1])

queuedLogger(logData[0])

def _subProc(self, iterNum, sharedObj, queue):
queue.put((f"In subproc {iterNum}", "debug"))
sharedObj.attr1 += iterNum

def run(self):
with QueueManager() as manager:
queue = manager.Queue()
shObj = manager.SharedObj()
_ = Process(target=self._loggingSubprocess, args=(queue,)).start()


pool = Pool(processes=cpu_count() - 2)
pool.starmap(
self._subProc,
zip(
range(10),
repeat(shObj),
repeat(queue),
)
)



def setupLogger():
logConfig = {
"format": "%(asctime)s %(filename)s->%(funcName)s():%(lineno)s %(levelname)s: %(message)s",
"datefmt": "%m/%d/%Y %I:%M:%S",
"encoding": "utf-8",
"loglevel": "DEBUG",
"warnings": True
}

logLevels = {
"CRITICAL": logging.CRITICAL,
"DEBUG": logging.DEBUG,
"ERROR": logging.ERROR,
"INFO": logging.INFO,
"WARNING": logging.WARNING,
}

logging.basicConfig(
filename="logs/logFile.log",
format=logConfig["format"],
datefmt=logConfig["datefmt"],
encoding=logConfig["encoding"],
level=logLevels[logConfig["loglevel"]],
)

logging.captureWarnings(logConfig["warnings"])

setupLogger()
if __name__ == "__main__":

Main().run()
</code></pre>
<p>but if I move the <code>setupLogger()</code> call to inside the <code>__main__</code> namescape it no longer works correctly.</p>
<pre><code>if __name__ == "__main__":
setupLogger()
Main().run()
</code></pre>
<p>Why can't the <code>setupLogger</code> call happen inside <code>__main__</code>?</p>
 

Latest posts

G
Replies
0
Views
1
Gamal Othman
G
Top