python-cmdb资产管理项目2-使用工厂模式模块化代码
概要:python-cmdb资产管理项目丰富中控指令,收集更多信息
1 工厂模式雏形
创建一个python项目,命令为autoclient,并创建一个名为lib的目录作为模块目录
整个目录结构如下:
1.1 模块代码
开发三个代码块,初步作为收集的信息
/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py
""" 用于采集硬盘信息 """ class DiskPlugin(object): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'Disk':'100G'}
/home/ningherui/PycharmProjects/autoclient/lib/plugins/memory.py
""" 用于采集内存信息 """ class MemoryPlugin(object): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'Memory':'1G'}
/home/ningherui/PycharmProjects/autoclient/lib/plugins/network.py
""" 用于采集网卡信息 """ class NetworkPlugin(object): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'IP':'192.168.100.101'}
1.2 使用配置文件setting.py
构建一个新的setting.py文件,作为配置文件
/home/ningherui/PycharmProjects/autoclient/setting.py
#作为一个配置文件 # PLUGIN_CLASS_LIST = [ # 'lib.plugins.disk.DiskPlugin', # 'lib.plugins.memory.MemoryPlugin', # 'lib.plugins.network.NetworkPlugin', # ] PLUGIN_CLASS_DICT = { "disk": 'lib.plugins.disk.DiskPlugin', "memory": 'lib.plugins.memory.MemoryPlugin', "network": 'lib.plugins.network.NetworkPlugin', }
主代码
/home/ningherui/PycharmProjects/autoclient/app.py
# 创建类库/模块目录lib # from lib.plugins.disk import DiskPlugin # from lib.plugins.memory import MemoryPlugin # from lib.plugins.network import NetworkPlugin # # def run(): # #实例化对象 # obj1 = DiskPlugin() # disk_info = obj1.process() # # obj2 = MemoryPlugin() # memory_info = obj2.process() # # obj3 = NetworkPlugin() # Network_info = obj3.process() # # if __name__ == '__main__': # run() import importlib from setting import PLUGIN_CLASS_DICT def run(): for key,path in PLUGIN_CLASS_DICT.items(): #key = "disk", path = "lib.plugins.disk.DiskPlugin" """ 使用下面的语句从右向左根据地一个点(.)分成两个字符串, lib.plugins.disk DiskPlugin """ module_path,class_name = path.rsplit('.',maxsplit=1) #根据一个字符串导入一个模块,需要使用importlib的模块 module = importlib.import_module(module_path) #等价于import lib.plugins.disk #获取一个类的对象 cls = getattr(module,class_name) #print(key,cls()) plugin_object = cls() info = plugin_object.process() print(key,info) if __name__ == '__main__': run()
执行
disk {'Disk': '100G'}
memory {'Memory': '1G'}
network {'IP': '192.168.100.101'}
基于反射和工厂模式实现可扩展
这样的方式有利于模块的扩展,当添加一个新的模块,只需要在setting的[配置文件加入相应的模块配置,就可以使用,删除也是如此
1.3 使用约束类,约束模块使用的方法
在上述的模块中都使用的process方法,但是当在添加的新的模块中,如果定义新的方法,不叫process,这时添加到setting的配置中,并不能生效,则可以定一个约束类
代码如下:
class BasePlugin(object): """ 基类,用于做约束.约束子类中必须实现process方法 """ def process(self): raise NotImplementedError("%s中必须实现Process方法" %self.__class__.__name__) class DiskPlugin(BasePlugin): pass class MemoryPlugin(BasePlugin): pass obj = DiskPlugin() obj.process()
执行后结果如下:
raise NotImplementedError("%s中必须实现Process方法" %self.__class__.__name__) NotImplementedError: DiskPlugin中必须实现Process方法
整理后代码如下
/home/ningherui/PycharmProjects/autoclient/lib/plugins/base.py
class BasePlugin(object): """ 基类,用于做约束.约束子类中必须实现process方法 """ def process(self): raise NotImplementedError("%s中必须实现Process方法" %self.__class__.__name__)
/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py
""" 用于采集硬盘信息 """ from .base import BasePlugin class DiskPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'Disk':'100G'}
/home/ningherui/PycharmProjects/autoclient/lib/plugins/memory.py
""" 用于采集内存信息 """ from .base import BasePlugin class MemoryPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'Memory':'1G'}
/home/ningherui/PycharmProjects/autoclient/lib/plugins/network.py
""" 用于采集网卡信息 """ from .base import BasePlugin class NetworkPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self): # 假设执行命令 return {'IP':'192.168.100.101'}
当模块的方法不是process就会报错,并会提示是那一个模块的方法错误
文件结构
把获取的信息进行汇总,单独放出来
建立一个__init__.py文件
/home/ningherui/PycharmProjects/autoclient/lib/plugins/__init__.py
import importlib import setting def get_server_info(): server_info = {} for key,path in setting.PLUGIN_CLASS_DICT.items(): #key = "disk", path = "lib.plugins.disk.DiskPlugin" """ 使用下面的语句从右向左根据地一个点(.)分成两个字符串, lib.plugins.disk DiskPlugin """ module_path,class_name = path.rsplit('.',maxsplit=1) #根据一个字符串导入一个模块,需要使用importlib的模块 module = importlib.import_module(module_path) #等价于import lib.plugins.disk #获取一个类的对象 cls = getattr(module,class_name) #print(key,cls()) plugin_object = cls() info = plugin_object.process() server_info[key] = info return server_info
主文件
/home/ningherui/PycharmProjects/autoclient/app.py
# 创建类库/模块目录lib # from lib.plugins.disk import DiskPlugin # from lib.plugins.memory import MemoryPlugin # from lib.plugins.network import NetworkPlugin # # def run(): # #实例化对象 # obj1 = DiskPlugin() # disk_info = obj1.process() # # obj2 = MemoryPlugin() # memory_info = obj2.process() # # obj3 = NetworkPlugin() # Network_info = obj3.process() # # if __name__ == '__main__': # run() import importlib import setting from lib.plugins import get_server_info def ssh(hostname,cmd): import paramiko private_key = paramiko.RSAKey.from_private_key_file('/home/ningherui/.ssh/id_rsa') # 创建SSH对象 ssh = paramiko.SSHClient() # 允许连接不在know_hosts文件中的主机 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 连接服务器 ssh.connect(hostname=hostname, port=setting.SSH_PORT, username=setting.SSH_USER, pkey=setting.SSH_KEY) # 执行命令 stdin, stdout, stderr = ssh.exec_command(cmd) # 获取命令结果 result = stdout.read() # 关闭连接 ssh.close() return result # def run(): # for key,path in setting.PLUGIN_CLASS_DICT.items(): # #key = "disk", path = "lib.plugins.disk.DiskPlugin" # """ # 使用下面的语句从右向左根据地一个点(.)分成两个字符串, # lib.plugins.disk DiskPlugin # """ # module_path,class_name = path.rsplit('.',maxsplit=1) # #根据一个字符串导入一个模块,需要使用importlib的模块 # module = importlib.import_module(module_path) #等价于import lib.plugins.disk # #获取一个类的对象 # cls = getattr(module,class_name) # #print(key,cls()) # plugin_object = cls() # info = plugin_object.process() # print(key,info) def run(): #获取__init__.py的get_server_info方法的到的数据 server_info = get_server_info() print(server_info) if __name__ == '__main__': run()
执行
/usr/bin/python3 /home/ningherui/PycharmProjects/autoclient/app.py {'disk': {'Disk': '100G'}, 'memory': {'Memory': '1G'}, 'network': {'IP': '192.168.100.101'}}
1.4 使用shh连接远程执行操作获得数据
代码如下:
主代码
/home/ningherui/PycharmProjects/autoclient/app.py
# 创建类库/模块目录lib # from lib.plugins.disk import DiskPlugin # from lib.plugins.memory import MemoryPlugin # from lib.plugins.network import NetworkPlugin # # def run(): # #实例化对象 # obj1 = DiskPlugin() # disk_info = obj1.process() # # obj2 = MemoryPlugin() # memory_info = obj2.process() # # obj3 = NetworkPlugin() # Network_info = obj3.process() # # if __name__ == '__main__': # run() import importlib import setting from lib.plugins import get_server_info def ssh(hostname,cmd): import paramiko private_key = paramiko.RSAKey.from_private_key_file('/home/ningherui/.ssh/id_rsa') # 创建SSH对象 ssh = paramiko.SSHClient() # 允许连接不在know_hosts文件中的主机 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 连接服务器 ssh.connect(hostname=hostname, port=setting.SSH_PORT, username=setting.SSH_USER, pkey=private_key) # 执行命令 stdin, stdout, stderr = ssh.exec_command(cmd) # 获取命令结果 result = stdout.read() # 关闭连接 ssh.close() return result # def run(): # for key,path in setting.PLUGIN_CLASS_DICT.items(): # #key = "disk", path = "lib.plugins.disk.DiskPlugin" # """ # 使用下面的语句从右向左根据地一个点(.)分成两个字符串, # lib.plugins.disk DiskPlugin # """ # module_path,class_name = path.rsplit('.',maxsplit=1) # #根据一个字符串导入一个模块,需要使用importlib的模块 # module = importlib.import_module(module_path) #等价于import lib.plugins.disk # #获取一个类的对象 # cls = getattr(module,class_name) # #print(key,cls()) # plugin_object = cls() # info = plugin_object.process() # print(key,info) def run(): #获取__init__.py的get_server_info方法的到的数据,获取两个参数 hostname = '192.168.100.101' server_info = get_server_info(ssh,hostname) print(server_info) if __name__ == '__main__': run()
__init__代码:
import importlib import setting def get_server_info(ssh,hostname): server_info = {} for key,path in setting.PLUGIN_CLASS_DICT.items(): #key = "disk", path = "lib.plugins.disk.DiskPlugin" """ 使用下面的语句从右向左根据地一个点(.)分成两个字符串, lib.plugins.disk DiskPlugin """ module_path,class_name = path.rsplit('.',maxsplit=1) #根据一个字符串导入一个模块,需要使用importlib的模块 module = importlib.import_module(module_path) #等价于import lib.plugins.disk #获取一个类的对象 cls = getattr(module,class_name) #print(key,cls()) plugin_object = cls() info = plugin_object.process(ssh,hostname) server_info[key] = info return server_info
模块代码:
-#base.py class BasePlugin(object): """ 基类,用于做约束.约束子类中必须实现process方法 """ def process(self,ssh,hostname): raise NotImplementedError("%s中必须实现Process方法" %self.__class__.__name__) #disk.py """ 用于采集硬盘信息 """ from .base import BasePlugin class DiskPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self,ssh,hostname): # 假设执行命令 # return {'Disk':'100G'} result = ssh(hostname,' df -H|grep root|awk \'{print $2}\'') return result.decode('utf-8') #network.py """ 用于采集网卡信息 """ from .base import BasePlugin class NetworkPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self,ssh,hostname): result = ssh(hostname, 'ifconfig ens33|grep -v inet6|grep inet|awk \'{print $2}\'') return result.decode('utf-8') #memory.py """ 用于采集内存信息 """ from .base import BasePlugin class MemoryPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self,ssh,hostname): result = ssh(hostname, "free -h |grep Mem|awk '{print $2}'") return result.decode('utf-8')
在setting.py中配置,可以在主代码中调用配置
SSH_USER='root' SSH_PORT='22'
执行结果
/usr/bin/python3 /home/ningherui/PycharmProjects/autoclient/app.py {'disk': '40G\n', 'memory': '1.9G\n', 'network': '192.168.100.101\n'}
1.5 收集多个机器信息
收集多个机器,并传入api,显示,api的代码参考上一章autoserver(https://www.cnblogs.com/zyxnhr/p/14391361.html)
# 创建类库/模块目录lib # from lib.plugins.disk import DiskPlugin # from lib.plugins.memory import MemoryPlugin # from lib.plugins.network import NetworkPlugin # # def run(): # #实例化对象 # obj1 = DiskPlugin() # disk_info = obj1.process() # # obj2 = MemoryPlugin() # memory_info = obj2.process() # # obj3 = NetworkPlugin() # Network_info = obj3.process() # # if __name__ == '__main__': # run() import importlib import requests import setting from lib.plugins import get_server_info def ssh(hostname,cmd): import paramiko private_key = paramiko.RSAKey.from_private_key_file('/home/ningherui/.ssh/id_rsa') # 创建SSH对象 ssh = paramiko.SSHClient() # 允许连接不在know_hosts文件中的主机 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 连接服务器 ssh.connect(hostname=hostname, port=setting.SSH_PORT, username=setting.SSH_USER, pkey=private_key) # 执行命令 stdin, stdout, stderr = ssh.exec_command(cmd) # 获取命令结果 result = stdout.read() # 关闭连接 ssh.close() return result # def run(): # for key,path in setting.PLUGIN_CLASS_DICT.items(): # #key = "disk", path = "lib.plugins.disk.DiskPlugin" # """ # 使用下面的语句从右向左根据地一个点(.)分成两个字符串, # lib.plugins.disk DiskPlugin # """ # module_path,class_name = path.rsplit('.',maxsplit=1) # #根据一个字符串导入一个模块,需要使用importlib的模块 # module = importlib.import_module(module_path) #等价于import lib.plugins.disk # #获取一个类的对象 # cls = getattr(module,class_name) # #print(key,cls()) # plugin_object = cls() # info = plugin_object.process() # print(key,info) def run(): #获取__init__.py的get_server_info方法的到的数据,获取两个参数 # hostname = '192.168.100.101' host_list = [ '192.168.100.101', '192.168.100.102', '192.168.100.103', ] for hostname in host_list: server_info = get_server_info(ssh,hostname) result = requests.post( url='http://127.0.0.1:8000/api/get_data', json={'host': hostname, 'info': server_info} ) # print(server_info) # server_info = get_server_info(ssh,hostname) # print(server_info) print('把资产信息发送到API', result.text) if __name__ == '__main__': run()
执行:
server端显示
2 使用线程池多线程
测试脚本
import time from concurrent.futures import ThreadPoolExecutor def task(i): time.sleep(1) print(i) pool = ThreadPoolExecutor(10) for i in range(87): pool.submit(task,i)
app.py脚本如下:
# 创建类库/模块目录lib # from lib.plugins.disk import DiskPlugin # from lib.plugins.memory import MemoryPlugin # from lib.plugins.network import NetworkPlugin # # def run(): # #实例化对象 # obj1 = DiskPlugin() # disk_info = obj1.process() # # obj2 = MemoryPlugin() # memory_info = obj2.process() # # obj3 = NetworkPlugin() # Network_info = obj3.process() # # if __name__ == '__main__': # run() import importlib import requests import setting from lib.plugins import get_server_info def ssh(hostname,cmd): import paramiko private_key = paramiko.RSAKey.from_private_key_file('/home/ningherui/.ssh/id_rsa') # 创建SSH对象 ssh = paramiko.SSHClient() # 允许连接不在know_hosts文件中的主机 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # 连接服务器 ssh.connect(hostname=hostname, port=setting.SSH_PORT, username=setting.SSH_USER, pkey=private_key) # 执行命令 stdin, stdout, stderr = ssh.exec_command(cmd) # 获取命令结果 result = stdout.read() # 关闭连接 ssh.close() return result # def run(): # for key,path in setting.PLUGIN_CLASS_DICT.items(): # #key = "disk", path = "lib.plugins.disk.DiskPlugin" # """ # 使用下面的语句从右向左根据地一个点(.)分成两个字符串, # lib.plugins.disk DiskPlugin # """ # module_path,class_name = path.rsplit('.',maxsplit=1) # #根据一个字符串导入一个模块,需要使用importlib的模块 # module = importlib.import_module(module_path) #等价于import lib.plugins.disk # #获取一个类的对象 # cls = getattr(module,class_name) # #print(key,cls()) # plugin_object = cls() # info = plugin_object.process() # print(key,info) def run(): #获取__init__.py的get_server_info方法的到的数据,获取两个参数 # hostname = '192.168.100.101' host_list = [ '192.168.100.101', '192.168.100.102', '192.168.100.103', ] for hostname in host_list: server_info = get_server_info(ssh,hostname) result = requests.post( url='http://127.0.0.1:8000/api/get_data', json={'host': hostname, 'info': server_info} ) # print(server_info) # server_info = get_server_info(ssh,hostname) # print(server_info) print('把资产信息发送到API', result.text) if __name__ == '__main__': run()
测试结果
server端 api显示:
3 单例模式使用
3.1单例模式示例
class Singletoton(object): instance = None #静态字段,类变量 def __init__(self,name): #初始化对象 self.name = name def __new__(cls, *args, **kwargs): #创建对象 if not cls.instance: cls.instance = object.__new__(cls) return cls.instance obj1 = Singletoton('alex') obj2 = Singletoton('erick') print(obj1) print(obj2)
执行结果
多线程导致创建多个对象
import time class Singletoton(object): instance = None #静态字段,类变量 def __init__(self,name): #初始化对象 self.name = name def __new__(cls, *args, **kwargs): #创建对象 if not cls.instance: time.sleep(0.2) cls.instance = object.__new__(cls) return cls.instance import threading def func(): obj = Singletoton('xxx') print(obj) for i in range(10): thread = threading.Thread(target=func) thread.start()
执行结果
原因:
是因为当进程执行到创建对象时,10个线程都执行这一步,但是前面的线程没有执行完,就导致创建新的对象
3.2 加锁解决多线程问题
import time import threading class Singletoton(object): instance = None #静态字段,类变量 lock = threading.RLock() def __init__(self,name): #初始化对象 self.name = name def __new__(cls, *args, **kwargs): #创建对象 with cls.lock: if not cls.instance: time.sleep(0.2) cls.instance = object.__new__(cls) return cls.instance def func(): obj = Singletoton('xxx') print(obj) for i in range(10): thread = threading.Thread(target=func) thread.start()
执行结果
这种条件下,当在并发多线程的过程中,当有线程外的进程创建对象时,就会有锁锁住,无法创建新的对象,效率比较低
3.3 代码优化
import time import threading class Singletoton(object): instance = None #静态字段,类变量 lock = threading.RLock() def __init__(self,name): #初始化对象 self.name = name def __new__(cls, *args, **kwargs): #创建对象 if cls.instance: return cls.instance with cls.lock: if not cls.instance: time.sleep(0.2) cls.instance = object.__new__(cls) return cls.instance def func(): obj = Singletoton('xxx') print(obj) for i in range(10): thread = threading.Thread(target=func) thread.start()
通过模块实现单例模式
# 文件1 # danli.py class Foo(object): pass site = Foo()
# 文件2 from danli.py import site print(site) #单例模式
4 日志处理(异常堆栈信息)
4.1 traceback模块
捕获异常的堆栈信息
import traceback try: int('dauchdasukd') except Exception as e: # print(e) print(traceback.format_exc())
执行看错误信息:
4.2 单个日志处理
import logging logging.basicConfig(filename='log.txt', format='%(asctime)s - %(name)s - %(levelname)s - %(module)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S %p', level=10) logging.error('xxxxxxxxl')
执行后,会生成log.txt文件,内容如下:
2021-02-14 11:16:23 AM - root - ERROR - test: xxxxxxxxl
对于上述记录日志的功能,只能将日志记录自爱单个文件中,如果要设置多个日志文件,logging.basicConfig将无法完成,需要自定义文件和日志操作对象
4.3 多日志处理
import logging file_haddler = logging.FileHandler('run.log','a',encoding='utf-8') fmt = logging.Formatter(fmt='%(asctime)s - %(name)s - %(levelname)s - %(module)s: %(message)s') file_haddler.setFormatter(fmt) file_haddler_1 = logging.FileHandler('run_1.log','a',encoding='utf-8') fmt = logging.Formatter() file_haddler_1.setFormatter(fmt) logger1 = logging.Logger('ssssxxxxx',level=logging.DEBUG) logger1.addHandler(file_haddler) logger1.addHandler(file_haddler_1)
logger1.error('asdzxcqwe')
执行会生成两个日志文件
4.4 引用日志文件
创建一个utils的公共库目录,在该目录下创建一个log模块的库,同时创建一个log目录,同时log的名字为cmdb.log
/home/ningherui/PycharmProjects/autoclient/lib/utils/log.py
import logging import setting class Logger(): def __init__(self,log_file_path,level): file_haddler = logging.FileHandler(log_file_path, 'a', encoding='utf-8') fmt = logging.Formatter(fmt='%(asctime)s - %(name)s - %(levelname)s - %(module)s: %(message)s') file_haddler.setFormatter(fmt) self.logger = logging.Logger('cmdb',level=level) self.logger.addHandler(file_haddler) def error(self,msg): self.logger.error(msg) #调用这个logger模块,写入日志 logger = Logger(setting.LOGGING_PATH,logging.DEBUG)
disk磁盘日志
/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py
""" 用于采集硬盘信息 """ from .base import BasePlugin from lib.utils.log import logger import traceback class DiskPlugin(BasePlugin): """ 采集硬盘信息 """ def process(self,ssh,hostname): result = None try: result = int('cdacas') except Exception as e: logger.error(traceback.format_exc()) # 假设执行命令 # return {'Disk':'100G'} #result = ssh(hostname,' df -H|grep root|awk \'{print $2}\'') # return result.decode('utf-8') return result
执行app后,生成的日志文件
/home/ningherui/PycharmProjects/autoclient/log/cmdb.log
2021-02-14 14:25:23,577 - cmdb - ERROR - log: Traceback (most recent call last): File "/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py", line 14, in process result = int('cdacas') ValueError: invalid literal for int() with base 10: 'cdacas' 2021-02-14 14:25:23,578 - cmdb - ERROR - log: Traceback (most recent call last): File "/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py", line 14, in process result = int('cdacas') ValueError: invalid literal for int() with base 10: 'cdacas' 2021-02-14 14:25:23,578 - cmdb - ERROR - log: Traceback (most recent call last): File "/home/ningherui/PycharmProjects/autoclient/lib/plugins/disk.py", line 14, in process result = int('cdacas') ValueError: invalid literal for int() with base 10: 'cdacas'
感谢老男孩教育