简单的apm性能收集方案

原理

本文通过浏览器提供的performance接口获取页面性能数据。performace相关的了解请查看performance。

整体技术方案

apm

浏览器上报sdk添加性能收集部分。

_performance: function (config) {
            var performance = window.performance;
            if (!!performance) {
                var data = {};
                data.origin_pathname = location.origin + location.pathname;
                if (performance.memory) {
                    data.jsHeapSizeLimit = performance.memory.jsHeapSizeLimit;
                    data.totalJSHeapSize = performance.memory.totalJSHeapSize;
                    data.usedJSHeapSize = performance.memory.usedJSHeapSize;
                    if (
                        data.jsHeapSizeLimit === 0 ||
                        data.totalJSHeapSize === 0 ||
                        data.usedJSHeapSize === 0
                    ) {
                        return false;
                    }
                }
                if (performance.navigation) {
                    data.redirectCount = performance.navigation.redirectCount;
                    data.type = performance.navigation.type;
                }
                // if(performance.timeOrigin) {
                //     data.timeOrigin = performance.timeOrigin;
                // }
                if (!!performance.timing) {
                    var navigationStart = performance.timing.navigationStart;
                    
                    var redirectStart = performance.timing.redirectStart; 
                    
                    var redirectEnd = performance.timing.redirectEnd;
                    
                    var fetchStart = performance.timing.fetchStart;
                    
                    var domainLookupStart = performance.timing.domainLookupStart;
                    
                    var domainLookupEnd = performance.timing.domainLookupEnd;
                    
                    var connectStart = performance.timing.connectStart;
                    
                    var secureConnectionStart = performance.timing.secureConnectionStart;
                    
                    var connectEnd = performance.timing.connectEnd;
                    
                    var requestStart = performance.timing.requestStart;
                    
                    var responseStart = performance.timing.responseStart;
                    
                    var responseEnd = performance.timing.responseEnd;
                    
                    var domLoading = performance.timing.domLoading;
                    
                    var domInteractive = performance.timing.domInteractive;
                    
                    var domContentLoadedEventStart = performance.timing.domContentLoadedEventStart;
                    
                    var domContentLoadedEventEnd = performance.timing.domContentLoadedEventEnd;  
                    
                    var domComplete = performance.timing.domComplete;
                    
                    var loadEventStart = performance.timing.loadEventStart;  
                    
                    var loadEventEnd = performance.timing.loadEventEnd;
                    
                    var unloadEventStart = performance.timing.unloadEventStart; 
                    
                    var unloadEventEnd = performance.timing.unloadEventEnd; 
                    
                    data.appCache = Math.max(domainLookupStart - fetchStart, 0);
                    data.dns = domainLookupEnd - domainLookupStart;
                    data.connection = connectEnd - connectStart;
                    data.request = responseStart - requestStart;
                    data.response = responseEnd - responseStart;
                    data.loading = responseEnd - requestStart;
                    data.rendering = domComplete - domLoading;
                    data.blankScreen = domContentLoadedEventEnd - navigationStart;
                    data.domComplete = domComplete - navigationStart;
                    data.loaded = loadEventEnd - navigationStart;
                    data.loadEvent = loadEventEnd - loadEventStart;

                    if (
                        data.appCache < 0 || data.appCache > 120000 || 
                        data.dns < 0 || data.dns > 120000 || 
                        data.connection < 0 || data.connection > 120000 || 
                        data.request < 0 || data.request > 120000 || 
                        data.loading < 0 || data.loading > 120000 || 
                        data.rendering < 0 || data.rendering > 120000 || 
                        data.blankScreen < 0 || data.blankScreen > 120000 || 
                        data.domComplete < 0 || data.domComplete > 120000 || 
                        data.loaded < 0 || data.loaded > 120000 || 
                        data.loadEvent < 0 || data.loadEvent > 120000
                    ) {
                        return false;
                    }
                }

                try {

                    var origin = config.url.split('/api');
                    var url = origin[0] + '/api/apm/performance';
                    this.request.ajax({
                        url: url || 'https://xxx.xxx.com/api/apm/performance',
                        type: 'post',
                        data: data,
                        success: function (res) {
                            //do nothing
                        }
                    });
                } catch (e) {
                    console && console.error(JSON.stringify(e));
                }

            }
        },

如上代码我们注意如几个点：

1. performance有兼容性问题存在，所以要先判断浏览器是否已支持。
2. 内存数据可能回有异常，如jsHeapSizeLimit=0。存在异常数据可过滤掉本条数据。
3. 性能数据每一条都可能有异常值，导致计算出来的差值可能小于零，也可能很大。这里做了一个正常值判断。0<=正常数据<=120,000ms。

服务器接性能数据

post: async (ctx, next) => {
        try {
            let reqQuery = ctx.request.body;
            let queryData = {
                origin_pathname: reqQuery['origin_pathname'],

                jsHeapSizeLimit: reqQuery['jsHeapSizeLimit'],
                totalJSHeapSize: reqQuery['totalJSHeapSize'],
                usedJSHeapSize: reqQuery['usedJSHeapSize'],

                redirectCount: reqQuery['redirectCount'],
                type: reqQuery['type'],

                redirect: reqQuery['redirect'],
                appCache: reqQuery['appCache'],
                dns: reqQuery['dns'],
                connection: reqQuery['connection'],
                request: reqQuery['request'],
                response: reqQuery['response'],

                loading: reqQuery['loading'],
                rendering: reqQuery['rendering'],
                blankScreen: reqQuery['blankScreen'],
                domComplete: reqQuery['domComplete'],
                loaded: reqQuery['loaded'],
                loadEvent: reqQuery['loadEvent'],
                create_time: moment().unix(),
            };

            if (
                queryData.appCache < 0 || queryData.appCache > 120000 ||
                queryData.dns < 0 || queryData.dns > 120000 ||
                queryData.connection < 0 || queryData.connection > 120000 ||
                queryData.request < 0 || queryData.request > 120000 ||
                queryData.response < 0 || queryData.response > 120000 ||
                queryData.loading < 0 || queryData.loading > 120000 ||
                queryData.rendering < 0 || queryData.rendering > 120000 ||
                queryData.blankScreen < 0 || queryData.blankScreen > 120000 ||
                queryData.domComplete < 0 || queryData.domComplete > 120000 ||
                queryData.loaded < 0 || queryData.domComplete > 120000 ||
                queryData.loadEvent < 0 || queryData.loadEvent > 120000 ||
                queryData.jsHeapSizeLimit == 0 || isNaN(queryData.jsHeapSizeLimit) ||
                queryData.totalJSHeapSize == 0 || isNaN(queryData.totalJSHeapSize) ||
                queryData.usedJSHeapSize == 0 || isNaN(queryData.usedJSHeapSize)
            ) {
                ctx.response.body = repWrapper({
                    status: "OK",
                    info: '不合理数据,暂不做统计'
                })
                return;
            }

            let status = await new Promise((resolve, reject) => {
                // rdsCli.keys('performance:*', function (err, keys) {
                //     let len = keys.length;
                //     if (len < 10416) {    //日300w pv 平均每5分钟的pv数
                rdsCli.set(`performance:${utils.guid()}`, JSON.stringify(queryData), 'EX', (70), function (err, status) {
                    if (err) {
                        reject(err);
                        throw err;
                    } else {
                        resolve({
                            status
                        });
                    }
                });
                //     } else {
                //         resolve({
                //             status: "OK",
                //             info: '达到设定的redis缓存上线，不做redis缓存处理'
                //         });
                //     }
                // });
            });

            ctx.response.body = repWrapper(status)
        } catch (e) {
            ctx.response.body = repWrapper(e.toString(), errMapping['ERR_DATABASE_REQ']['code']);
            throw e;
        }
    }

此部分将数据缓存到redis70s。同理这里也做了一些数据校验，过滤掉无用的数据。

定时任务获取redis中的数据并入库

module.exports = async () => {
    let data = await new Promise((resolve, reject) => {
        rdsCli.keys('performance:*', async function (err, keys) {
            let data = await bluebird.map(keys, async (v, k) => {
                let value = await getAsync(v);
                return JSON.parse(
                    value
                );
            });
            _.map(keys, (key) => {
                rdsCli.set(key, '', 'EX', 1);
            });
            resolve(data);
        });
    });

    //计算
    let archive = _.groupBy(data, 'origin_pathname');

    let archiveData = Object.keys(archive).map((v) => {
        return archive[v].reduce((a, b) => {
            let temp = {};
            _.keys(a).forEach((sv) => {
                if ('origin_pathname' === sv) {
                    temp[sv] = a[sv];
                } else if( 'create_time' === sv ) {
                    temp[sv] = Math.round( ( parseFloat(a[sv]) + parseFloat(b[sv]) ) / 2)
                } else {
                    temp[sv] = ( ( parseFloat(a[sv]) + parseFloat(b[sv]) ) / 2).toFixed(2);
                }
            });
            return temp;
        })
    });

    archiveData = archiveData.filter((v) => {
        return v.jsHeapSizeLimit != 'NaN' && 
        v.totalJSHeapSize != 'NaN' && 
        v.usedJSHeapSize != 'NaN' &&
        v.domComplete != 'NaN' && 
        v.loadEvent != 'NaN';
    });

    //批量插入数据库
    let queryRst = await db.elPerformance.bulkCreate(archiveData);
    
}

几个需要注意的点

1. 从redis拿数据是30秒一次。而redis的缓存是70秒，所以拿到redis数据后，需要将已经拿到的数据删除，以防出现重复数据。
2. 这里做的不好的点从redis拿出的数据校验不够完善，需优化。

获取性能数据

get: async (ctx, next) => {
        try {
            // 取参
            let {
                origin_pathname,
                start_time,
                end_time
            } = ctx.query;

            // 查询
            let whereQuery = {};
            const Op = db.sequelize.Op;
            if (origin_pathname) {
                
                whereQuery['origin_pathname'] = {
                    [Op.like]: `%${origin_pathname}%`
                };
            }
            if (start_time && end_time) {
                whereQuery['create_time'] = {
                    $gt: start_time,
                    $lt: end_time
                };
            }

            let queryRst = await db.elPerformance.findAndCountAll({
                attributes: [
                    db.sequelize.fn('DISTINCT', db.sequelize.col('origin_pathname')),
                ],
                where: whereQuery,
                raw: true
            });

            // 返回
            ctx.response.body = repWrapper({
                list: queryRst.rows,
                page: {
                    current: 1 || current,
                    per_page: 15 || per_page,
                    total_page: 15 || Math.ceil(queryRst.count.length / per_page)
                }
            });
        } catch (e) {
            ctx.response.body = repWrapper(e.toString(), errMapping['ERR_DATABASE_REQ']['code']);
            throw e;
        }
    },

    one: async (ctx, next) => {
        let origin_pathname = decodeURIComponent(ctx.params.id);
        try {
            // 取参
            let {
                start_time,
                end_time,
            } = ctx.query;

            // 查询
            let whereQuery = {};
            whereQuery['origin_pathname'] = origin_pathname;
            if (start_time && end_time) {
                whereQuery['create_time'] = {
                    $gt: start_time,
                    $lt: end_time
                };
            }

            let queryDetail = await db.elPerformance.findAndCountAll({
                attributes: [
                    'connection',
                    'request',
                    'response',
                    'loading',
                    'rendering',
                    'blankScreen',
                    'domComplete',
                    'loaded',
                    'loadEvent',
                    'create_time'
                ],
                where: whereQuery
            });
            let qdf = {
                'connection': [],
                'request': [],
                'response': [],
                'loading': [],
                'rendering': [],
                'blankScreen': [],
                'domComplete': [],
                'loaded': [],
                'loadEvent': [],
                'create_time': []
            };
            queryDetail.rows.forEach((v) => {
                let data = v.dataValues;
                Object.keys(data).forEach( sv => {
                    if( 'create_time'=== sv ) {
                        qdf[sv].push( moment.unix(v[sv]).format("YYYY-MM-DD hh:MM:SS") );
                    } else {
                        qdf[sv].push(v[sv]);
                    }
                });
            });
            

            // 返回
            ctx.response.body = repWrapper({
                origin_pathname,
                queryDetail: qdf,
                count: queryDetail.count,
            });
        } catch (e) {
            ctx.response.body = repWrapper(e.toString(), errMapping['ERR_DATABASE_REQ']['code']);
            throw e;
        }
    },

这一步就很简单了，先获取列表的数据，再获取具体某一条链接的性能数据。这里也有几个注意点

1. 默认设置的查询时间比较短（一天）
2. 没有一次性计算完所有页面的数据，计算量太大，假设每条连接每天有8000条数据，500条链接总共就会有4000万条数据，数据砍半也有2000万条数据。很好性能。所以改成了如上的查询方式。
3. 对某些数据字段添加了索引（id, origin_path, create_at），加速查询。

后台界面展示

1. 列表页

   

   list

2. 具体性能页面

   

   detail