2 files changed, 18 insertions(+), 12 deletions(-)

M README.md
M app/Console/Commands/MonitorProbe.php

@@ 12,6 12,8 @@ Some probes are implemented by external commands, and those command will likely 
 
 After setting up the environment and running database migrations, interacts the application with artisan `monitor:*` subcommands (see them with `php artisan list`). Currently all administrative usage is in CLI. See `--help` of those subcommands for more information. Register your probes with `monitor:register`.
 
+Set `MONITOR_PROBE_MAX_TRIES` (defaults to 1) in `.env` if you want probes to retry when failed.
+
 `monitor:probe` should be run once every hour. Set up a cron job for it.
 
 The served web page will list the status for all probes, under user-facing string descibing the monitored service specified by you instead of detailed description. Currently 7 days of historical data will be visualized.

@@ 42,19 42,23 @@ class MonitorProbe extends Command
     {
         foreach (ProbeInstance::all() as $probe_instance) {
             $probe = unserialize($probe_instance->probe);
-            try {
-                $probe->execute();
-                $log = new ProbeLog();
-                $log->success = true;
-                $probe_instance->logs()->save($log);
-            } catch (Exception $e) {
-                $this->error($probe->describe() . ' failed');
-                $this->info($e->getMessage());
-                $log = new ProbeLog();
-                $log->success = false;
-                $log->outputs = $e->getMessage();
-                $probe_instance->logs()->save($log);
+            $success = false;
+            $lastlog = '';
+            for ($i = env('MONITOR_PROBE_MAX_TRIES', 1); $i > 0; $i--) {
+                try {
+                    $probe->execute();
+                    $success = true;
+                    break;
+                } catch (Exception $e) {
+                    $this->error("{$probe->describe()} failed, $i tries left");
+                    $this->info($e->getMessage());
+                    $lastlog = $e->getMessage();
+                }
             }
+            $log = new ProbeLog();
+            $log->success = $success;
+            $log->outputs = $lastlog;
+            $probe_instance->logs()->save($log);
         }
     }
 }