一、CANN 日志基础1.1 日志级别CANN 日志级别 (从低到高): DEBUG → 调试信息开发阶段使用 INFO → 一般信息默认级别 WARNING → 警告信息不影响运行 ERROR → 错误信息功能受损 FATAL → 致命错误程序终止1.2 基础配置# 环境变量配置exportASCEND_LOG_LEVEL3# INFO 级别exportASCEND_LOG_PATH/var/log/ascendexportASCEND_LOG_FILEascend.logexportASCEND_LOG_FILE_SIZE100# MBexportASCEND_LOG_FILE_NUM10# 保留文件数# 运行时查看日志tail-f/var/log/ascend/ascend.log1.3 Python 日志接口importlogging# 获取 CANN 日志器loggerlogging.getLogger(ascend)# 配置日志格式handlerlogging.StreamHandler()formatterlogging.Formatter([%(asctime)s] [%(levelname)s] [%(name)s] %(message)s,datefmt%Y-%m-%d %H:%M:%S)handler.setFormatter(formatter)logger.addHandler(handler)logger.setLevel(logging.INFO)# 使用日志logger.info(模型加载开始)logger.warning(显存使用率超过 80%)logger.error(模型转换失败)二、自定义日志2.1 推理日志装饰器importfunctoolsimporttimeimportloggingdeflog_inference(func):推理日志装饰器functools.wraps(func)defwrapper(*args,**kwargs):loggerlogging.getLogger(inference)start_timetime.time()logger.info(f开始推理:{func.__name__})try:resultfunc(*args,**kwargs)latencytime.time()-start_time logger.info(f推理完成:{func.__name__}f耗时{latency:.3f}s f输入形状{args[0].shapeifhasattr(args[0],shape)elseN/A})returnresultexceptExceptionase:latencytime.time()-start_time logger.error(f推理失败:{func.__name__}f耗时{latency:.3f}s f错误{str(e)})raisereturnwrapper# 使用示例log_inferencedefpredict(model,input_data):returnmodel(input_data)2.2 性能追踪日志classPerformanceTracer:def__init__(self,log_fileperformance.jsonl):self.log_filelog_file self.loggerlogging.getLogger(performance)deftrace(self,operation_name):性能追踪上下文管理器returnTraceContext(self,operation_name)defrecord(self,operation_name,duration,metadataNone):记录性能数据record{timestamp:time.time(),operation:operation_name,duration_ms:duration*1000,metadata:metadataor{}}# 写入 JSONL 文件withopen(self.log_file,a)asf:f.write(json.dumps(record) )# 同时输出到日志self.logger.info(f性能:{operation_name}f耗时{duration*1000:.2f}ms f元数据{metadata})classTraceContext:def__init__(self,tracer,operation_name):self.tracertracer self.operation_nameoperation_name self.start_timeNonedef__enter__(self):self.start_timetime.time()returnselfdef__exit__(self,exc_type,exc_val,exc_tb):durationtime.time()-self.start_time metadata{}ifexc_type:metadata[error]str(exc_val)self.tracer.record(self.operation_name,duration,metadata)returnFalse# 使用示例tracerPerformanceTracer()withtracer.trace(model_load):modelload_model(model.om)withtracer.trace(preprocess):input_datapreprocess(raw_data)withtracer.trace(inference):outputmodel(input_data)withtracer.trace(postprocess):resultpostprocess(output)三、分布式日志收集3.1 日志聚合importloggingimportlogging.handlersimportsocketclassDistributedLogHandler(logging.Handler):分布式日志处理器def__init__(self,host,port,node_id):super().__init__()self.hosthost self.portport self.node_idnode_id self.socketsocket.socket(socket.AF_INET,socket.SOCK_STREAM)self.socket.connect((host,port))defemit(self,record):发送日志到中心服务器try:log_entry{node_id:self.node_id,timestamp:record.created,level:record.levelname,message:self.format(record),module:record.module,function:record.funcName,line:record.lineno}messagejson.dumps(log_entry).encode(utf-8)self.socket.send(message)exceptExceptionase:self.handleError(record)# 配置分布式日志defsetup_distributed_logging(node_id,log_server192.168.1.100,port5000):配置分布式日志loggerlogging.getLogger()logger.setLevel(logging.INFO)# 控制台输出console_handlerlogging.StreamHandler()console_handler.setFormatter(logging.Formatter([%(asctime)s] [%(node_id)s] [%(levelname)s] %(message)s))logger.addHandler(console_handler)# 分布式输出distributed_handlerDistributedLogHandler(log_server,port,node_id)distributed_handler.setFormatter(logging.Formatter([%(asctime)s] [%(node_id)s] [%(levelname)s] [%(module)s:%(funcName)s:%(lineno)d] %(message)s))logger.addHandler(distributed_handler)returnlogger3.2 ELK 日志栈# docker-compose-elk.ymlversion:3.8services:elasticsearch:image:docker.elastic.co/elasticsearch/elasticsearch:8.10.0environment:-discovery.typesingle-node-xpack.security.enabledfalseports:-9200:9200volumes:-esdata:/usr/share/elasticsearch/datalogstash:image:docker.elastic.co/logstash/logstash:8.10.0volumes:-./logstash.conf:/usr/share/logstash/pipeline/logstash.confports:-5000:5000depends_on:-elasticsearchkibana:image:docker.elastic.co/kibana/kibana:8.10.0ports:-5601:5601depends_on:-elasticsearchvolumes:esdata:# logstash.confinput{tcp{port5000codecjson_lines}}filter{# 添加字段mutate{add_field{servicecann-inference}}# 解析时间戳date{match[timestamp,UNIX_MS]}}output{elasticsearch{hosts[elasticsearch:9200]indexcann-logs-%{YYYY.MM.dd}}}四、基于日志的性能分析4.1 日志分析脚本importjsonimportpandasaspdfromcollectionsimportdefaultdictclassLogAnalyzer:def__init__(self,log_file):self.log_filelog_file self.recordsself._parse_log()def_parse_log(self):解析日志文件records[]withopen(self.log_file,r)asf:forlineinf:try:recordjson.loads(line.strip())records.append(record)exceptjson.JSONDecodeError:continuereturnrecordsdefperformance_summary(self):性能摘要dfpd.DataFrame(self.records)ifduration_msnotindf.columns:print(无性能数据)returnsummarydf.groupby(operation)[duration_ms].agg([count,mean,std,min,max,lambdax:x.quantile(0.5),lambdax:x.quantile(0.95),lambdax:x.quantile(0.99)])summary.columns[次数,平均(ms),标准差,最小(ms),最大(ms),P50(ms),P95(ms),P99(ms)]print( 性能摘要:)print(summary.to_string())returnsummarydefbottleneck_analysis(self):瓶颈分析dfpd.DataFrame(self.records)ifduration_msnotindf.columns:return# 按操作统计operation_statsdf.groupby(operation)[duration_ms].agg([mean,sum,count])operation_statsoperation_stats.sort_values(sum,ascendingFalse)print( 瓶颈分析(按总耗时排序):)print(operation_stats.to_string())# 识别瓶颈total_timeoperation_stats[sum].sum()cumulative0bottlenecks[]forop,rowinoperation_stats.iterrows():cumulativerow[sum]percentagecumulative/total_time*100bottlenecks.append((op,row[sum],percentage))ifpercentage80:# 80/20 法则breakprint(f 前{len(bottlenecks)}个操作占总耗时80%:)forop,total,pctinbottlenecks:print(f{op}:{total:.2f}ms ({pct:.1f}%))# 使用示例analyzerLogAnalyzer(performance.jsonl)analyzer.performance_summary()analyzer.bottleneck_analysis()4.2 实时监控classRealtimeLogMonitor:def__init__(self,log_file):self.log_filelog_file self.metricsdefaultdict(list)self.lockthreading.Lock()defstart(self):启动实时监控self.threadthreading.Thread(targetself._monitor_loop,daemonTrue)self.thread.start()def_monitor_loop(self):监控循环withopen(self.log_file,r)asf:f.seek(0,2)# 跳到文件末尾whileTrue:linef.readline()ifnotline:time.sleep(0.1)continuetry:recordjson.loads(line.strip())self._process_record(record)exceptjson.JSONDecodeError:continuedef_process_record(self,record):处理日志记录withself.lock:ifduration_msinrecord:oprecord.get(operation,unknown)self.metrics[op].append(record[duration_ms])# 保持最近 1000 条iflen(self.metrics[op])1000:self.metrics[op]self.metrics[op][-1000:]defget_current_metrics(self):获取当前指标withself.lock:metrics{}forop,durationsinself.metrics.items():ifdurations:metrics[op]{count:len(durations),avg:sum(durations)/len(durations),p99:sorted(durations)[int(len(durations)*0.99)]}returnmetrics# 使用示例monitorRealtimeLogMonitor(performance.jsonl)monitor.start()# 定期获取指标whileTrue:time.sleep(10)metricsmonitor.get_current_metrics()print(json.dumps(metrics,indent2))五、常见问题问题原因解决方案日志文件过大日志级别太低调整为 INFO 或 WARNING日志缺失未正确配置路径检查 ASCEND_LOG_PATH性能日志格式错误JSON 解析失败检查日志格式分布式日志延迟网络问题增加缓冲区、异步发送日志文件损坏异常终止使用日志轮转相关仓库CANN- 昇腾异构计算架构 https://atomgit.com/cannge- 图引擎 https://atomgit.com/cann/gecann-recipes-infer- 推理配方 https://atomgit.com/cann/ops-nnruntime- 运行时 https://atomgit.com/cann/runtimecann-samples- 示例代码 https://atomgit.com/cann-sampleshccl- 集合通信库 https://atomgit.com/cann/hccl
