Zeroconf offline-handling improvements (#77)

* Add benchmark.py

* Add cpu / disk benchmark APIs

* Add cpu_benchmark method to distributed_job_manager.py

* Do a better job of storing hostnames =

* Remove hostname from Zeroconf cache if server goes offline

* Add cpu / disk benchmark APIs

* Add cpu_benchmark method to distributed_job_manager.py

* Do a better job of storing hostnames =

* Remove hostname from Zeroconf cache if server goes offline

* Wrap main code in try finally block to always stop zeroconf

* Add missing import

src/api/server_proxy.py

@@ -10,6 +10,7 @@ from requests_toolbelt.multipart import MultipartEncoder, MultipartEncoderMonito
 
 from src.utilities.misc_helper import is_localhost
 from src.utilities.status_utils import RenderStatus
+from src.utilities.zeroconf_server import ZeroconfServer
 
 status_colors = {RenderStatus.ERROR: "red", RenderStatus.CANCELLED: 'orange1', RenderStatus.COMPLETED: 'green',
                  RenderStatus.NOT_STARTED: "yellow", RenderStatus.SCHEDULED: 'purple',
@@ -19,7 +20,7 @@ categories = [RenderStatus.RUNNING, RenderStatus.WAITING_FOR_SUBJOBS, RenderStat
               RenderStatus.SCHEDULED, RenderStatus.COMPLETED, RenderStatus.CANCELLED, RenderStatus.UNDEFINED]
 
 logger = logging.getLogger()
-OFFLINE_MAX = 2
+OFFLINE_MAX = 4
 LOOPBACK = '127.0.0.1'
 
 
@@ -87,6 +88,13 @@ class RenderServerProxy:
             self.__offline_flags = self.__offline_flags + 1
         except Exception as e:
             logger.exception(f"Uncaught exception: {e}")
+
+        # If server unexpectedly drops off the network, remove from Zeroconf list
+        if self.__offline_flags > OFFLINE_MAX:
+            try:
+                ZeroconfServer.client_cache.pop(self.hostname)
+            except KeyError:
+                pass
         return None
 
     def request(self, payload, timeout=5):