Background: In large systems, online exceptions are inevitable, so how can we detect the application running problems as early as possible and deal with them in time so as not to expand the impact?

The answer is to monitor the application and alert it for errors.

This paper uses SpringBoot + Dingding to realize application monitoring alarm, of course, can also use enterprise wechat, SMS notification and so on

Dingding development platform – custom robot access documents

Looking at the document of the nail, it can be found that the function of message notification can be realized by calling Webhook address to send the alarm message to the group chat

Implementation steps

Get call address

  • Start by creating a spike group and create a custom robot

Select one of the security Settings to enhance security and prevent Webhook address leaks from being spam messages

  • Created successfully, copy the Webhook address, you will need it later

Example Creating a SpringBoot application

  • The project structure

WarnService

Define the interface for reporting error messages

Error messages are added to the MonitorMessageQueue queue

interface WarnService {
    fun reportErrorMsg(moduleName: String, msg: String)

}

@Service
class WarnServiceImpl : WarnService {
    @Autowired
    private lateinit var monitorMessageQueue: MonitorMessageQueue
    override fun reportErrorMsg(moduleName: String, msg: String) {
        monitorMessageQueue.add(MessageDto().apply {
            this.moduleName = moduleName
            this.content = msg
            this.timestamp = System.currentTimeMillis()
        })
    }
Copy the code

MonitorMessageQueue queue

Methods provided by queues

  • Start: starts the daemon thread
  • Drain: wait for timeout to return to the queue element. In this case, set timeout to 30 seconds
  • Add: Adds elements to the queue

@Component
@Scope("singleton")
class MonitorMessageQueue {
    private val queue: BlockingQueue<MessageDto> = LinkedBlockingQueue()
    private val logger = LoggerFactory.getLogger(MonitorMessageQueue::class.java)

    @Autowired
    private lateinit var sendService: DataSendService

    @PostConstruct
    private fun start(a) {
        logger.info("MonitorMessageQueue start")
        val thread = Thread(DataSendThread(this, sendService), "monitor_thread_0")
        thread.isDaemon = true
        thread.start()
    }

    // Each robot can send up to 20 messages to the group per minute. If more than 20 messages are sent, the stream will be restricted for 10 minutes.
    fun drain(a): ArrayList<MessageDto> {
        val bulkData = ArrayList<MessageDto>()
        Queues.drain(queue, bulkData, Int.MAX_VALUE, 30, TimeUnit.SECONDS)
        return bulkData
    }

    fun add(message: MessageDto) {
        queue.add(message)
    }
}
Copy the code

The alarm thread

Monitor the MonitorMessageQueue queue, group and summarize the messages, and invoke the send service to send the messages


class DataSendThread(private val queue: MonitorMessageQueue, private val sendService: DataSendService) : Runnable {
    private val sendCount = AtomicLong(0)
    private val stop = false
    private val logger = LoggerFactory.getLogger(DataSendThread::class.java)
    override fun run(a) {
        while(! stop) {val list = queue.drain()
            if (list.isNullOrEmpty()) {
                logger.info("queue isEmpty")
                return
            }
            val format = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
            val mid = UUID.randomUUID().toString().replace("-"."")
            val stringBuilder = StringBuilder("[${format.format(System.currentTimeMillis())}[APP monitoring alarm]")
            stringBuilder.append("\n");
            list.groupBy { it.moduleName }.map {
                stringBuilder.append("${it.key}(${it.value.size}Time)")
                stringBuilder.append("\n") stringBuilder.append(it.value.firstOrNull()? .content ? :"")
                stringBuilder.append("\n")
            }
            stringBuilder.append("Http://127.0.0.1/monitor/detail? mid=${mid}")
            sendService.send(stringBuilder.toString())
            logger.info("send success:${sendCount.addAndGet(1)}")}}}Copy the code

DataSendService

Process the signature and actually call Webhook to send the alarm information to the nail group chat


interface DataSendService {
    fun send(content: String)
}

@Service
class DataSendServiceImpl : DataSendService {
    @Autowired
    private lateinit var restTemplate: RestTemplate

    private fun url(timestamp: Long, sign: String): String {
        return "https://oapi.dingtalk.com/robot/send?access_token=xxxxxxx&timestamp=${timestamp}&sign=$sign"

    }

    override fun send(content: String) {
        val timestamp = System.currentTimeMillis()
        println(
            restTemplate.postForObject(
                url(timestamp, calcSign(timestamp)), mapOf(
                    "msgtype" to "text"."text" to mapOf(
                        "content" to content
                    )
                ),
                String::class.java
            )
        )
    }


    private fun calcSign(timestamp: Long): String {
        val secret = "xxxxxxx"

        val stringToSign = "" "$timestamp
            $secret"" ".trimIndent()
        val mac = Mac.getInstance("HmacSHA256")
        mac.init(SecretKeySpec(secret.toByteArray(charset("UTF-8")), "HmacSHA256"))
        val signData = mac.doFinal(stringToSign.toByteArray(charset("UTF-8")))
        return URLEncoder.encode(String(Base64.getEncoder().encode(signData)), "UTF-8")}}Copy the code

Run the test case


@SpringBootTest
internal class WarnServiceImplTest {
    @Autowired
    private lateinit var warnService: WarnService

    @Test
    fun reportErrorMsg(a) {
        while (true) {
            for (i in 1.. ((1000 * Math.random()).toInt())) {
                warnService.reportErrorMsg("app-test1"."too many error")}for (i in (1.. ((1000 * Math.random()).toInt()))) {
                warnService.reportErrorMsg("app-test2"."too many error")
            }
            Thread.sleep(1000 * 30)}}}Copy the code