From 62d4b8ac039a0d4fc107f6fa337dff15d1655f15 Mon Sep 17 00:00:00 2001
From: xdavidwu <xdavidwuph@gmail.com>
Date: Fri, 30 Jul 2021 21:27:21 +0800
Subject: [PATCH] cli: monitor:probe: impl optional retry

---
 README.md                             |  2 ++
 app/Console/Commands/MonitorProbe.php | 28 +++++++++++++++------------
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index c65cd33..a97f2a2 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,8 @@ Some probes are implemented by external commands, and those command will likely
 
 After setting up the environment and running database migrations, interacts the application with artisan `monitor:*` subcommands (see them with `php artisan list`). Currently all administrative usage is in CLI. See `--help` of those subcommands for more information. Register your probes with `monitor:register`.
 
+Set `MONITOR_PROBE_MAX_TRIES` (defaults to 1) in `.env` if you want probes to retry when failed.
+
 `monitor:probe` should be run once every hour. Set up a cron job for it.
 
 The served web page will list the status for all probes, under user-facing string descibing the monitored service specified by you instead of detailed description. Currently 7 days of historical data will be visualized.
diff --git a/app/Console/Commands/MonitorProbe.php b/app/Console/Commands/MonitorProbe.php
index 5f32e74..8c4e70d 100644
--- a/app/Console/Commands/MonitorProbe.php
+++ b/app/Console/Commands/MonitorProbe.php
@@ -42,19 +42,23 @@ class MonitorProbe extends Command
     {
         foreach (ProbeInstance::all() as $probe_instance) {
             $probe = unserialize($probe_instance->probe);
-            try {
-                $probe->execute();
-                $log = new ProbeLog();
-                $log->success = true;
-                $probe_instance->logs()->save($log);
-            } catch (Exception $e) {
-                $this->error($probe->describe() . ' failed');
-                $this->info($e->getMessage());
-                $log = new ProbeLog();
-                $log->success = false;
-                $log->outputs = $e->getMessage();
-                $probe_instance->logs()->save($log);
+            $success = false;
+            $lastlog = '';
+            for ($i = env('MONITOR_PROBE_MAX_TRIES', 1); $i > 0; $i--) {
+                try {
+                    $probe->execute();
+                    $success = true;
+                    break;
+                } catch (Exception $e) {
+                    $this->error("{$probe->describe()} failed, $i tries left");
+                    $this->info($e->getMessage());
+                    $lastlog = $e->getMessage();
+                }
             }
+            $log = new ProbeLog();
+            $log->success = $success;
+            $log->outputs = $lastlog;
+            $probe_instance->logs()->save($log);
         }
     }
 }
-- 
2.43.0