查异常重启 | Notion

查看日志配置


pm2 log --lines 500

cat /root/.pm2/logs/main-out-0.log | grep 'Starting Nest application...'

cat application-2024-01-18.log.21 | grep 'Starting Nest application...'

grep -A 50 -B 50 'Starting Nest application...' /root/.pm2/logs/main-out-0.log

grep -B 200 'Starting Nest application...' /root/.pm2/logs/main-out-0.log

grep -B 50 'Starting Nest application...' /root/.pm2/logs/main-out-0.log

Untitled

解决方案

异步异常未处理
fastify基础框架有问题已经升级解决
Authguard 403 返回有问题，后面返回 true false 会抛二次异常，出现问题
Fastify session 有问题，memjs 保存应该是异步，大部分情况下没问题，这个改一下回调判断，或者直接返回 setcookie 都行，肯定是有副作用的，但是比重启强；找到错误链路 try catch 不抛出异常

疑惑

nestjs Exception 和 Error 不分家？比如和 mysql、redis、memCache 连接超时报 NodeError？

node崩溃怎么办

try catche 解决
uncaughtException unhandledRejection 处理
pm2 守护进程

Untitled

错误链路

Error: Cannot set headers after they are sent to the client
    at new NodeError (node:internal/errors:399:5)
    at ServerResponse.setHeader (node:_http_outgoing:645:11)
    at ServerResponse.writeHead (node:_http_server:378:21)
    at safeWriteHead (/data/srv/node_modules/fastify/lib/reply.js:556:9)
    at onSendEnd (/data/srv/node_modules/fastify/lib/reply.js:595:5)
    at wrapOnSendEnd (/data/srv/node_modules/fastify/lib/reply.js:549:5)
    at next (/data/srv/node_modules/fastify/lib/hooks.js:293:7)
    at Object.onSend (/data/srv/node_modules/@fastify/compress/index.js:229:14)
    at next (/data/srv/node_modules/fastify/lib/hooks.js:299:30)
    at /data/srv/node_modules/aiy-fastify-session/lib/fastifySession.js:199:11
    at /data/srv/node_modules/aiy-fastify-session/lib/session.js:180:11
    at /data/srv/node_modules/connect-memjs-with-php-serialize/lib/connect-memjs.js:183:22
    at /data/srv/node_modules/memjs/lib/memjs/memjs.js:239:23
    at responseHandler (/data/srv/node_modules/memjs/lib/memjs/memjs.js:832:21)
    at Server.respond (/data/srv/node_modules/memjs/lib/memjs/server.js:45:3)
    at Server.responseHandler (/data/srv/node_modules/memjs/lib/memjs/server.js:106:12) {
  code: 'ERR_HTTP_HEADERS_SENT'
}
Error: Premature close
    at new NodeError (node:internal/errors:399:5)
    at Duplexify.onclose (node:internal/streams/end-of-stream:149:30)
    at Duplexify.emit (node:events:525:35)
    at Duplexify._destroy (/data/srv/node_modules/duplexify/index.js:199:8)
    at /data/srv/node_modules/duplexify/index.js:182:10
    at processTicksAndRejections (node:internal/process/task_queues:77:11) {
  code: 'ERR_STREAM_PREMATURE_CLOSE'
}

----

0|main     | (node:17) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
0|main     | (Use `node --trace-deprecation ...` to show where the warning was created)
0|main     | TypeError: cb is not a function
0|main     |     at fallbackErrorHandler (/data/srv/node_modules/fastify/lib/error-handler.js:127:3)
0|main     |     at handleError (/data/srv/node_modules/fastify/lib/error-handler.js:61:5)
0|main     |     at onErrorHook (/data/srv/node_modules/fastify/lib/reply.js:781:5)
0|main     |     at wrapOnSendEnd (/data/srv/node_modules/fastify/lib/reply.js:547:5)
0|main     |     at next (/data/srv/node_modules/fastify/lib/hooks.js:301:7)
0|main     |     at /data/srv/node_modules/aiy-fastify-session/lib/fastifySession.js:199:11
0|main     |     at /data/srv/node_modules/aiy-fastify-session/lib/session.js:180:11
0|main     |     at /data/srv/node_modules/connect-memjs-with-php-serialize/lib/connect-memjs.js:183:22
0|main     |     at /data/srv/node_modules/memjs/lib/memjs/memjs.js:239:23
0|main     |     at responseHandler (/data/srv/node_modules/memjs/lib/memjs/memjs.js:832:21)
0|main     |     at Server.respond (/data/srv/node_modules/memjs/lib/memjs/server.js:45:3)
0|main     |     at Server.responseHandler (/data/srv/node_modules/memjs/lib/memjs/server.js:106:12)
0|main     |     at Socket.<anonymous> (/data/srv/node_modules/memjs/lib/memjs/server.js:140:14)
0|main     |     at Socket.emit (node:events:513:28)
0|main     |     at addChunk (node:internal/streams/readable:324:12)
0|main     |     at readableAddChunk (node:internal/streams/readable:297:9)
0|main     | (node:87) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
0|main     | (Use `node --trace-deprecation ...` to show where the warning was created)

修正后错误问题

没看到容器有重启

反而是 pm2 的重启

helper1.5

ai2.0


# pm2 error log
0|main     | (node:17) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
0|main     | (Use `node --trace-deprecation ...` to show where the warning was created)
0|main     | (node:2771) [DEP0005] DeprecationWarning: Buffer() is deprecated due to security and usability issues. Please use the Buffer.alloc(), Buffer.allocUnsafe(), or Buffer.from() methods instead.
0|main     | (Use `node --trace-deprecation ...` to show where the warning was created)
# deployment resource limit
resources:
          requests: 
            cpu: 0.1
            memory: "300Mi" 
          limits:
            cpu: 1.5
            memory: "1.5Gi"
# pm2 config
module.exports = {
    apps: [
        {
            script: 'dist/src/main.js',
            watch: false,
            instances: 1,
            autorestart: true,
            max_memory_restart: '4G',
            max_size: '100M', // 设置日志文件的最大大小
            keep_logs: 1, // 保留的归档日志文件数量
            env: { NODE_ENV: 'production' },
        },
    ],
};

function onSend (options) {
    const cookieOpts = options.cookie
    const saveUninitializedSession = options.saveUninitialized
    const rollingSessions = options.rolling

    return function saveSession (request, reply, payload, done) {
      const session = request.session
      if (!session || !session.sessionId || !session.encryptedSessionId) {
        done()
        return
      }

      const cookieSessionId = getCookieSessionId(request)
      const saveSession = shouldSaveSession(request, cookieSessionId, saveUninitializedSession, rollingSessions)
      const isInsecureConnection = cookieOpts.secure === true && isConnectionSecure(request) === false
      const sessionIdWithPrefix = hasCookiePrefix ? `${cookiePrefix}${session.encryptedSessionId}` : session.encryptedSessionId
      if (!saveSession || isInsecureConnection) {
        // if a session cookie is set, but has a different ID, clear it
        if (cookieSessionId && cookieSessionId !== session.encryptedSessionId) {
          reply.clearCookie(cookieName, { domain: cookieOpts.domain })
        }

        if (session.isSaved()) {
          if(options.env !== 'development'){
            if (options.headerName) {
              reply.header(options.headerName, session.sessionId)
            }

            reply.setCookie(
              cookieName,
              sessionIdWithPrefix,
              // we need to remove extra properties to align the same with `express-session`
              session.cookie.toJSON()
            )
          }
        }
        done()
        return
      }

      if(options.env !== 'development'){
        if (options.headerName) {
          reply.header(options.headerName, session.sessionId)
        }

        reply.setCookie(
          cookieName,
          sessionIdWithPrefix,
          // we need to remove extra properties to align the same with `express-session`
          session.cookie.toJSON()
        )
      }

      session.save((err) => {
        if (err) {
          done(err)
          return
        }
        // async save
        done()
      })
    }
  }

修正过后效果

Untitled

经过近一周观察，内驱趋于稳定

Untitled

启动二周仍无重启，服务指标正常

Untitled

20231208

20240128