ping handling and free up job

main.go

@@ -75,6 +75,7 @@ func main() {
 
 	robot.LoadRobotsFromDatabase()
 	go robot.RobotPingHandler()
+	go robot.RobotFreeUpJobHandler()
 
 	app.Listen(config.Cfg.Host + ":" + config.Cfg.Port)
 }

modules/robot/robot.go

@@ -29,17 +29,20 @@ func RobotPingHandler() {
 
 			if err != nil {
 				logger.AddSystemLog(rslogger.LogTypeWarning, "Robot ping request failed: %v", err)
+				logger.AddRobotLog(rslogger.LogTypeWarning, robot.Type, robot.Id, "Robot ping request failed: %v", err)
 
 				if robot.PingRetries == utils.RobotPingRetries {
 					UpdateRobotStatus(robot, utils.RobotStatusOffline)
 
 					logger.AddSystemLog(rslogger.LogTypeWarning, "Robot %s marked as offline because %v attempts have already been made to reach it", robot.Name, utils.RobotPingRetries)
+					logger.AddRobotLog(rslogger.LogTypeWarning, robot.Type, robot.Id, "Robot marked as offline because %v attempts have already been made to reach it", utils.RobotPingRetries)
 					continue
 				}
 
 				robot.PingRetries++
 
 				logger.AddSystemLog(rslogger.LogTypeWarning, "Trying to ping robot %s for %v times failed", robot.Name, robot.PingRetries)
+				logger.AddRobotLog(rslogger.LogTypeWarning, robot.Type, robot.Id, "Trying to ping robot for %v times failed", robot.PingRetries)
 
 				if robot.Status != utils.RobotStatusConnecting {
 					UpdateRobotStatus(robot, utils.RobotStatusConnecting)
@@ -48,7 +51,26 @@ func RobotPingHandler() {
 			}
 
 			if robot.Status == utils.RobotStatusConnecting {
-				UpdateRobotStatus(robot, utils.RobotStatusIdle)
+				if robot.CurrentJobId != "" {
+					UpdateRobotStatus(robot, utils.RobotStatusWaiting)
+				} else {
+					UpdateRobotStatus(robot, utils.RobotStatusIdle)
+				}
+
+				robot.ConnectedAt = time.Now()
+
+				BroadcastSSEMessage(structs.SSEMessage{
+					Cmd: utils.SSESentCmdRobotUpdated,
+					Body: struct {
+						RobotId     string
+						ConnectedAt time.Time
+					}{
+						RobotId:     robot.Id,
+						ConnectedAt: robot.ConnectedAt,
+					},
+				})
+
+				logger.AddRobotLog(rslogger.LogTypeInfo, robot.Type, robot.Id, "Robot connection established by ping")
 			}
 
 			if robot.PingRetries != 0 {
@@ -58,6 +80,36 @@ func RobotPingHandler() {
 	}
 }
 
+// This function will free up the job of a robot if the last task
+// of a job is older than the given duration (job not finished by requester)
+func RobotFreeUpJobHandler() {
+	ticker := time.NewTicker(utils.RobotFreeUpJobInterval)
+
+	for range ticker.C {
+		for _, robot := range cache.GetRobots() {
+			if robot.Status == utils.RobotStatusOffline ||
+				robot.CurrentJobId == "" ||
+				robot.LastTaskAt.IsZero() ||
+				time.Since(robot.LastTaskAt) < utils.RobotFreeUpJobInterval ||
+				robot.Status == utils.RobotStatusError &&
+					robot.Status != utils.RobotStatusWaiting {
+				continue
+			}
+
+			logger.AddSystemLog(rslogger.LogTypeError, "Robot %s job with id %s and name %s freed up because it was not correctly finished", robot.Name, robot.CurrentJobId, robot.CurrentJobName)
+			logger.AddRobotLog(rslogger.LogTypeError, robot.Type, robot.Id, "Job with id %s and name %s freed up because it was not correctly finished", robot.CurrentJobId, robot.CurrentJobName)
+
+			FreeUpJob(robot, utils.RobotStatusError)
+		}
+	}
+}
+
+func FreeUpJob(robot *structs.Robot, newStatus uint8) {
+	UpdateRobotCurrentJob(robot, "", "")
+	UpdateRobotStatus(robot, newStatus)
+	CountDownJobsWaiting(robot)
+}
+
 func LoadRobotsFromDatabase() {
 	var robots []structs.Robot
 
@@ -91,6 +143,8 @@ func UpdateRobotStatus(robot *structs.Robot, status uint8) {
 			Status:  status,
 		},
 	})
+
+	logger.AddRobotLog(rslogger.LogTypeInfo, robot.Type, robot.Id, "Robot status updated to %s", utils.GetRobotStatusString(status))
 }
 
 func ProcessJobTask(r *structs.Robot, jobId string, jobName string) {
@@ -102,12 +156,13 @@ func ProcessJobTask(r *structs.Robot, jobId string, jobName string) {
 		CountUpJobsWaiting(r, jobId, jobName)
 	}
 
-	for r.CurrentJobId != "" && r.CurrentJobId != jobId {
+	for r.CurrentJobId != "" &&
+		r.CurrentJobId != jobId &&
+		r.Status != utils.RobotStatusWaiting &&
+		r.Status != utils.RobotStatusError {
 		// wait for current job to finish
-
 		if r.Status == utils.RobotStatusOffline {
-			log.Info().Msgf("Robot %s offline. Job with id %s and name %s canceled", r.Name, jobId, jobName)
-			logger.AddRobotLog(rslogger.LogTypeWarning, r.Type, r.Id, "Job with id %s and name %s canceled", jobId, jobName)
+			logger.AddRobotLog(rslogger.LogTypeError, r.Type, r.Id, "Job with id %s and name %s canceled", jobId, jobName)
 			return
 		}
 

modules/utils/globals.go

@@ -1,8 +1,11 @@
 package utils
 
+import "time"
+
 const (
 	RobotPingRetries            = 3
 	RobotPingHandlerInterval    = 5 // seconds
+	RobotFreeUpJobInterval      = 5 * time.Minute
 	RobotsPageLimit             = 5
 	RobotsPermitJoinAutoDisable = 120 // seconds
 	UnauthorizedRobotsPageLimit = 5

modules/utils/utils.go

@@ -25,6 +25,8 @@ func GetRobotStatusString(s uint8) string {
 		return "error"
 	case RobotStatusOffline:
 		return "offline"
+	case RobotStatusWaiting:
+		return "waiting"
 	default:
 		return "unknown"
 	}

routers/api/v1/control/control.go

@@ -7,6 +7,7 @@ import (
 	"jannex/robot-control-manager/modules/robot"
 	"jannex/robot-control-manager/modules/structs"
 	"jannex/robot-control-manager/modules/utils"
+	"time"
 
 	"git.ex.umbach.dev/Alex/roese-utils/rslogger"
 	"git.ex.umbach.dev/Alex/roese-utils/rsutils"
@@ -70,6 +71,8 @@ func ControlRex(c *fiber.Ctx) error {
 	robot.ProcessJobTask(r, body.JobId, body.JobName)
 	robot.UpdateRobotStatus(r, utils.RobotStatusRunning)
 
+	r.LastTaskAt = time.Now()
+
 	logger.AddRobotLog(rslogger.LogTypeInfo, utils.RobotTypeRex, r.Id, "Control robot with task: %v", rsutils.MarshalJson(body.Task))
 
 	if err := request.Request(fiber.MethodPost, r.Address+":5000/api/v1/control", body.Task); err != nil {

routers/api/v1/robot/robot.go

@@ -393,11 +393,9 @@ func FreeUpJob(c *fiber.Ctx) error {
 		return c.SendStatus(fiber.StatusUnprocessableEntity)
 	}
 
-	robot.UpdateRobotCurrentJob(foundRobot, "", "")
-	robot.UpdateRobotStatus(foundRobot, utils.RobotStatusIdle)
-	robot.CountDownJobsWaiting(foundRobot)
+	robot.FreeUpJob(foundRobot, utils.RobotStatusIdle)
 
-	logger.AddSystemLog(rslogger.LogTypeInfo, "Job freed up from robot %s", params.RobotId)
+	logger.AddRobotLog(rslogger.LogTypeWarning, foundRobot.Type, foundRobot.Id, "Job freed up")
 
 	return c.SendStatus(fiber.StatusOK)
 }

testclient/testrobot.py

@@ -33,7 +33,8 @@ def randomId():
 # 10 random rexrobot
 
 RexRobot("test")
-
+RexRobot("759")
+'''
 for i in range(1):
     #RexRobot(str(randomId()))
     RexRobot("759")
@@ -42,7 +43,7 @@ for i in range(1):
     RexRobot("762")
     RexRobot("763")
     RexRobot("764")
-    RexRobot("770")
+    RexRobot("770")'''