diff --git a/code/common/message-queue/src/main/java/nu/marginalia/mqsm/ActorStateMachine.java b/code/common/message-queue/src/main/java/nu/marginalia/mqsm/ActorStateMachine.java index a3f7edbe..cac47c6a 100644 --- a/code/common/message-queue/src/main/java/nu/marginalia/mqsm/ActorStateMachine.java +++ b/code/common/message-queue/src/main/java/nu/marginalia/mqsm/ActorStateMachine.java @@ -59,9 +59,9 @@ public class ActorStateMachine { registerStates(stateGraph); isDirectlyInitializable = stateGraph.isDirectlyInitializable(); - for (var declaredState : stateGraph.declaredStates()) { - if (!allStates.containsKey(declaredState.name())) { - throw new IllegalArgumentException("State " + declaredState.name() + " is not defined in the state graph"); + stateGraph.declaredStates().forEach((name, declaredState) -> { + if (!allStates.containsKey(name)) { + throw new IllegalArgumentException("State " + name + " is not defined in the state graph"); } if (!allStates.containsKey(declaredState.next())) { throw new IllegalArgumentException("State " + declaredState.next() + " is not defined in the state graph"); @@ -71,7 +71,7 @@ public class ActorStateMachine { throw new IllegalArgumentException("State " + state + " is not defined in the state graph"); } } - } + }); resume(); diff --git a/code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/AbstractStateGraph.java b/code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/AbstractStateGraph.java index 977f2ce4..648aad63 100644 --- a/code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/AbstractStateGraph.java +++ b/code/common/message-queue/src/main/java/nu/marginalia/mqsm/graph/AbstractStateGraph.java @@ -18,6 +18,9 @@ public abstract class AbstractStateGraph { this.stateFactory = stateFactory; } + /** User-facing description of the actor. */ + public abstract String describe(); + public void transition(String state) { throw new ControlFlowException(state, null); } @@ -30,7 +33,6 @@ public abstract class AbstractStateGraph { throw new ControlFlowException("ERROR", ""); } - public void error(T payload) { throw new ControlFlowException("ERROR", payload); } @@ -54,13 +56,13 @@ public abstract class AbstractStateGraph { return false; } - public Set declaredStates() { - Set ret = new HashSet<>(); + public Map declaredStates() { + Map ret = new HashMap<>(); for (var method : getClass().getMethods()) { var gs = method.getAnnotation(GraphState.class); if (gs != null) { - ret.add(gs); + ret.put(gs.name(), gs); } } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java index d9002e18..63e487d8 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/Actor.java @@ -3,7 +3,7 @@ package nu.marginalia.control.actor; public enum Actor { CRAWL, RECRAWL, - RECONVERT_LOAD, + CONVERT_AND_LOAD, CONVERTER_MONITOR, LOADER_MONITOR, CRAWLER_MONITOR, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java index cf88e683..a42aec50 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/ControlActors.java @@ -35,7 +35,7 @@ public class ControlActors { GsonFactory gsonFactory, BaseServiceParams baseServiceParams, ConvertActor convertActor, - ReconvertAndLoadActor reconvertAndLoadActor, + ConvertAndLoadActor convertAndLoadActor, CrawlActor crawlActor, RecrawlActor recrawlActor, ConverterMonitorActor converterMonitorFSM, @@ -56,7 +56,7 @@ public class ControlActors { register(Actor.CRAWL, crawlActor); register(Actor.RECRAWL, recrawlActor); register(Actor.CONVERT, convertActor); - register(Actor.RECONVERT_LOAD, reconvertAndLoadActor); + register(Actor.CONVERT_AND_LOAD, convertAndLoadActor); register(Actor.CONVERTER_MONITOR, converterMonitorFSM); register(Actor.LOADER_MONITOR, loaderMonitor); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java index 92bbc1d6..312e71d9 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/AbstractProcessSpawnerActor.java @@ -40,6 +40,10 @@ public class AbstractProcessSpawnerActor extends AbstractStateGraph { private final ProcessService.ProcessId processId; private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + public String describe() { + return "Spawns a(n) " + processId + " process and monitors its inbox for messages"; + } + @Inject public AbstractProcessSpawnerActor(StateFactory stateFactory, MqPersistence persistence, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java index 9f2ced26..190d1daf 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/FileStorageMonitorActor.java @@ -34,6 +34,11 @@ public class FileStorageMonitorActor extends AbstractStateGraph { private static final String END = "END"; private final FileStorageService fileStorageService; + @Override + public String describe() { + return "Monitor the file storage directories and purge any file storage area that has been marked for deletion," + + " and remove any file storage area that is missing from disk."; + } @Inject public FileStorageMonitorActor(StateFactory stateFactory, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java index 77384b06..1665a524 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/MessageQueueMonitorActor.java @@ -20,6 +20,10 @@ public class MessageQueueMonitorActor extends AbstractStateGraph { private static final String END = "END"; private final MqPersistence persistence; + @Override + public String describe() { + return "Periodically run maintenance tasks on the message queue"; + } @Inject public MessageQueueMonitorActor(StateFactory stateFactory, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java index 4128f6f9..e7542a4c 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/monitor/ProcessLivenessMonitorActor.java @@ -34,6 +34,11 @@ public class ProcessLivenessMonitorActor extends AbstractStateGraph { this.heartbeatService = heartbeatService; } + @Override + public String describe() { + return "Periodically check to ensure that the control service's view of running processes is agreement with the process heartbeats table."; + } + @GraphState(name = INITIAL, next = MONITOR) public void init() { } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java index 0bcc5293..f1e4b881 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertActor.java @@ -52,6 +52,11 @@ public class ConvertActor extends AbstractStateGraph { public long loaderMsgId = 0L; }; + @Override + public String describe() { + return "Convert a set of crawl data into a format suitable for loading into the database."; + } + @Inject public ConvertActor(StateFactory stateFactory, ActorProcessWatcher processWatcher, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ReconvertAndLoadActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java similarity index 94% rename from code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ReconvertAndLoadActor.java rename to code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java index 06c982ff..970e85d5 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ReconvertAndLoadActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ConvertAndLoadActor.java @@ -30,7 +30,7 @@ import java.nio.file.Files; import java.nio.file.StandardCopyOption; @Singleton -public class ReconvertAndLoadActor extends AbstractStateGraph { +public class ConvertAndLoadActor extends AbstractStateGraph { // STATES @@ -63,13 +63,18 @@ public class ReconvertAndLoadActor extends AbstractStateGraph { public long loaderMsgId = 0L; }; + @Override + public String describe() { + return "Process a set of crawl data and then load it into the database."; + } + @Inject - public ReconvertAndLoadActor(StateFactory stateFactory, - ActorProcessWatcher processWatcher, - ProcessOutboxes processOutboxes, - FileStorageService storageService, - IndexClient indexClient, - Gson gson + public ConvertAndLoadActor(StateFactory stateFactory, + ActorProcessWatcher processWatcher, + ProcessOutboxes processOutboxes, + FileStorageService storageService, + IndexClient indexClient, + Gson gson ) { super(stateFactory); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java index 40f447c1..bc26624d 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlActor.java @@ -46,6 +46,11 @@ public class CrawlActor extends AbstractStateGraph { public long crawlerMsgId = 0L; }; + @Override + public String describe() { + return "Run the crawler with the given crawl spec using no previous crawl data for a reference"; + } + @Inject public CrawlActor(StateFactory stateFactory, ProcessOutboxes processOutboxes, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java index 9cadc49a..1b611ce0 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/CrawlJobExtractorActor.java @@ -52,6 +52,11 @@ public class CrawlJobExtractorActor extends AbstractStateGraph { public record CrawlJobExtractorArguments(String description) { } public record CrawlJobExtractorArgumentsWithURL(String description, String url) { } + @Override + public String describe() { + return "Run the crawler job extractor process"; + } + @GraphState(name = CREATE_FROM_LINK, next = END, resume = ResumeBehavior.ERROR, description = """ diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java index 10227dc9..b82b2278 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/ExportDataActor.java @@ -48,6 +48,11 @@ public class ExportDataActor extends AbstractStateGraph { public FileStorageId storageId = null; }; + @Override + public String describe() { + return "Export data from the database to a storage area of type EXPORT."; + } + @Inject public ExportDataActor(StateFactory stateFactory, FileStorageService storageService, diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java index c4253a0d..071e61db 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/RecrawlActor.java @@ -46,6 +46,10 @@ public class RecrawlActor extends AbstractStateGraph { public long crawlerMsgId = 0L; }; + @Override + public String describe() { + return "Run the crawler with the given crawl spec using previous crawl data for a reference"; + } public static RecrawlMessage recrawlFromCrawlData(FileStorageId crawlData) { return new RecrawlMessage(null, crawlData, 0L); } diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java index 7441b437..ca78d871 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TriggerAdjacencyCalculationActor.java @@ -32,6 +32,11 @@ public class TriggerAdjacencyCalculationActor extends AbstractStateGraph { this.processService = processService; } + @Override + public String describe() { + return "Calculate website similarities"; + } + @GraphState(name = INITIAL, next = END, resume = ResumeBehavior.ERROR, description = """ diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java index 355620e5..90d449c3 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/actor/task/TruncateLinkDatabase.java @@ -33,6 +33,11 @@ public class TruncateLinkDatabase extends AbstractStateGraph { public FileStorageId storageId = null; }; + @Override + public String describe() { + return "Remove all data from the link database."; + } + @Inject public TruncateLinkDatabase(StateFactory stateFactory, HikariDataSource dataSource) diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java index 152af472..805038cb 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorRunState.java @@ -1,6 +1,11 @@ package nu.marginalia.control.model; -public record ActorRunState(String name, String state, boolean terminal, boolean canStart) { +public record ActorRunState(String name, + String state, + String actorDescription, + String stateDescription, + boolean terminal, + boolean canStart) { public String stateIcon() { if (terminal) { return "\uD83D\uDE34"; diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java index a9d7b783..34caef6f 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/model/ActorStateGraph.java @@ -7,17 +7,17 @@ import nu.marginalia.mqsm.state.MachineState; import java.util.*; import java.util.stream.Collectors; -public record ActorStateGraph(List states) { +public record ActorStateGraph(String description, List states) { public ActorStateGraph(AbstractStateGraph graph, MachineState currentState) { - this(getStateList(graph, currentState)); + this(graph.describe(), getStateList(graph, currentState)); } private static List getStateList( AbstractStateGraph graph, MachineState currentState) { - Map declaredStates = graph.declaredStates().stream().collect(Collectors.toMap(GraphState::name, gs -> gs)); + Map declaredStates = graph.declaredStates(); Set seenStates = new HashSet<>(declaredStates.size()); LinkedList edge = new LinkedList<>(); diff --git a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java index ddfbbe58..e883310a 100644 --- a/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java +++ b/code/services-core/control-service/src/main/java/nu/marginalia/control/svc/ControlActorService.java @@ -4,17 +4,20 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import nu.marginalia.control.actor.ControlActors; import nu.marginalia.control.actor.task.CrawlJobExtractorActor; -import nu.marginalia.control.actor.task.ReconvertAndLoadActor; +import nu.marginalia.control.actor.task.ConvertAndLoadActor; import nu.marginalia.control.actor.task.RecrawlActor; import nu.marginalia.control.actor.Actor; import nu.marginalia.control.model.ActorRunState; import nu.marginalia.control.model.ActorStateGraph; import nu.marginalia.db.storage.model.FileStorageId; +import nu.marginalia.mqsm.graph.GraphState; import nu.marginalia.mqsm.state.MachineState; import spark.Request; import spark.Response; import java.util.Comparator; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; @Singleton public class ControlActorService { @@ -64,7 +67,7 @@ public class ControlActorService { } public Object triggerProcessing(Request request, Response response) throws Exception { controlActors.start( - Actor.RECONVERT_LOAD, + Actor.CONVERT_AND_LOAD, FileStorageId.parse(request.params("fid")) ); return ""; @@ -75,24 +78,45 @@ public class ControlActorService { // Start the FSM from the intermediate state that triggers the load controlActors.startFrom( - Actor.RECONVERT_LOAD, - ReconvertAndLoadActor.LOAD, - new ReconvertAndLoadActor.Message(null, fid, 0L, 0L) + Actor.CONVERT_AND_LOAD, + ConvertAndLoadActor.LOAD, + new ConvertAndLoadActor.Message(null, fid, 0L, 0L) ); return ""; } + private final ConcurrentHashMap actorStateDescriptions = new ConcurrentHashMap<>(); + public Object getActorStates() { return controlActors.getActorStates().entrySet().stream().map(e -> { + final var stateGraph = controlActors.getActorDefinition(e.getKey()); + final MachineState state = e.getValue(); + final String actorDescription = stateGraph.describe(); + final String machineName = e.getKey().name(); final String stateName = state.name(); + + final String stateDescription = actorStateDescriptions.computeIfAbsent( + (machineName + "." + stateName), + k -> Optional.ofNullable(stateGraph.declaredStates().get(stateName)) + .map(GraphState::description) + .orElse("Description missing for " + stateName) + ); + + + final boolean terminal = state.isFinal(); final boolean canStart = controlActors.isDirectlyInitializable(e.getKey()) && terminal; - return new ActorRunState(machineName, stateName, terminal, canStart); + return new ActorRunState(machineName, + stateName, + actorDescription, + stateDescription, + terminal, + canStart); }) .filter(s -> !s.terminal() || s.canStart()) .sorted(Comparator.comparing(ActorRunState::name)) diff --git a/code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb b/code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb index d3d807e6..2f57488b 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/actor-details.hdb @@ -9,6 +9,7 @@ {{> control/partials/nav}}

{{actor}}

+

{{state-graph.description}}

{{> control/partials/actor-state-graph}} {{> control/partials/message-queue-table}}
diff --git a/code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb b/code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb index a09e16a4..fc6785e8 100644 --- a/code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb +++ b/code/services-core/control-service/src/main/resources/templates/control/partials/actors-table.hdb @@ -7,8 +7,8 @@ {{#each actors}} - {{name}} - {{stateIcon}} {{state}} + {{name}} + {{stateIcon}} {{state}} {{#unless terminal}}