diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java index 62d57aea..6acbaea6 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/configuration/ServiceDescriptor.java @@ -13,7 +13,6 @@ import nu.marginalia.wmsa.memex.MemexMain; import nu.marginalia.wmsa.podcasts.PodcastScraperMain; import nu.marginalia.wmsa.renderer.RendererMain; import nu.marginalia.wmsa.resource_store.ResourceStoreMain; -import nu.marginalia.wmsa.smhi.scraper.SmhiScraperMain; import org.apache.logging.log4j.core.lookup.MainMapLookup; import java.util.Map; @@ -26,7 +25,6 @@ public enum ServiceDescriptor { AUTH("auth", 5003, AuthMain.class), API("api", 5004, ApiMain.class), - SMHI_SCRAPER("smhi-scraper",5012, SmhiScraperMain.class), PODCST_SCRAPER("podcast-scraper", 5013, PodcastScraperMain.class), EDGE_INDEX("edge-index", 5021, EdgeIndexMain.class), diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java index 004a677d..fb3d0e9e 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java @@ -1,22 +1,15 @@ package nu.marginalia.wmsa.renderer; -import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.name.Named; -import nu.marginalia.wmsa.client.GsonFactory; import nu.marginalia.wmsa.configuration.server.Initialization; import nu.marginalia.wmsa.configuration.server.MetricsServer; import nu.marginalia.wmsa.configuration.server.Service; import nu.marginalia.wmsa.resource_store.ResourceStoreClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class RendererService extends Service { - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Gson gson = GsonFactory.get(); - private final ResourceStoreClient resourceStoreClient; @@ -24,7 +17,6 @@ public class RendererService extends Service { public RendererService(ResourceStoreClient resourceStoreClient, @Named("service-host") String ip, @Named("service-port") Integer port, - SmhiRendererService smhiRendererService, PodcastRendererService podcastRendererService, StatusRendererService statusRendererService, Initialization initialization, @@ -34,7 +26,6 @@ public class RendererService extends Service { this.resourceStoreClient = resourceStoreClient; - smhiRendererService.start(); podcastRendererService.start(); statusRendererService.start(); } diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/SmhiRendererService.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/SmhiRendererService.java deleted file mode 100644 index 56e4ff07..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/SmhiRendererService.java +++ /dev/null @@ -1,82 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import com.google.gson.Gson; -import com.google.inject.Inject; -import lombok.SneakyThrows; -import nu.marginalia.wmsa.client.GsonFactory; -import nu.marginalia.wmsa.configuration.server.Context; -import nu.marginalia.wmsa.renderer.mustache.MustacheRenderer; -import nu.marginalia.wmsa.renderer.mustache.RendererFactory; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq; -import nu.marginalia.wmsa.resource_store.ResourceStoreClient; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import nu.marginalia.wmsa.smhi.model.PrognosData; -import nu.marginalia.wmsa.smhi.model.index.IndexPlatser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.time.LocalDateTime; -import java.util.concurrent.TimeUnit; - -public class SmhiRendererService { - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Gson gson = GsonFactory.get(); - - private final RendererFactory rendererFactory = new RendererFactory(); - - private final MustacheRenderer indexRenderer; - private final MustacheRenderer prognosRenderer; - - private final ResourceStoreClient resourceStoreClient; - - - @Inject @SneakyThrows - public SmhiRendererService(ResourceStoreClient resourceStoreClient) { - this.resourceStoreClient = resourceStoreClient; - indexRenderer = rendererFactory.renderer( "smhi/index"); - prognosRenderer = rendererFactory.renderer( "smhi/prognos"); - } - - public void start() { - Spark.post("/render/smhi/index", this::renderSmhiIndex); - Spark.post("/render/smhi/prognos", this::renderSmhiPrognos); - } - - - private Object renderSmhiIndex(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, RenderSmhiIndexReq.class); - - logger.info("renderSmhiIndex()"); - var resource = new RenderedResource("index.html", - LocalDateTime.MAX, - indexRenderer.render(new IndexPlatser(req.platser))); - - resourceStoreClient.putResource(Context.fromRequest(request), "smhi", resource) - .timeout(10, TimeUnit.SECONDS) - .blockingSubscribe(); - - return ""; - } - - private Object renderSmhiPrognos(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, RenderSmhiPrognosReq.class); - - logger.info("renderSmhiPrognos({})", req.data.plats.namn); - var resource = new RenderedResource(req.data.plats.getUrl(), - LocalDateTime.now().plusHours(3), - prognosRenderer.render(req.data)); - - resourceStoreClient.putResource(Context.fromRequest(request), "smhi", resource) - .timeout(10, TimeUnit.SECONDS) - .blockingSubscribe(); - - return ""; - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java index e398f8b7..63537e3b 100644 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java +++ b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java @@ -11,8 +11,6 @@ import nu.marginalia.wmsa.podcasts.model.Podcast; import nu.marginalia.wmsa.podcasts.model.PodcastEpisode; import nu.marginalia.wmsa.podcasts.model.PodcastListing; import nu.marginalia.wmsa.podcasts.model.PodcastNewEpisodes; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq; import javax.inject.Inject; import java.util.concurrent.TimeUnit; @@ -24,19 +22,6 @@ public class RendererClient extends AbstractDynamicClient{ super(ServiceDescriptor.RENDERER); } - @SneakyThrows - public Observable render(Context ctx, RenderSmhiPrognosReq req) { - return post(ctx, "/render/smhi/prognos", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderSmhiPrognos()"))); - } - - - @SneakyThrows - public Observable render(Context ctx, RenderSmhiIndexReq req) { - return post(ctx, "/render/smhi/index", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderSmhiIndex()"))); - } - @SneakyThrows public Observable render(Context ctx, PodcastNewEpisodes req) { return post(ctx, "/render/podcast/new", req) diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiIndexReq.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiIndexReq.java deleted file mode 100644 index d585d56f..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiIndexReq.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.wmsa.renderer.request.smhi; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import nu.marginalia.wmsa.smhi.model.Plats; - -import java.util.List; - -@NoArgsConstructor @AllArgsConstructor @Getter -public class RenderSmhiIndexReq { - public List platser; -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiPrognosReq.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiPrognosReq.java deleted file mode 100644 index ba1746db..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/renderer/request/smhi/RenderSmhiPrognosReq.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.wmsa.renderer.request.smhi; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import nu.marginalia.wmsa.smhi.model.PrognosData; - -@NoArgsConstructor @AllArgsConstructor @Getter -public class RenderSmhiPrognosReq { - public PrognosData data; -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/SmhiScraperService.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/SmhiScraperService.java deleted file mode 100644 index 2b074efb..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/SmhiScraperService.java +++ /dev/null @@ -1,79 +0,0 @@ -package nu.marginalia.wmsa.smhi; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.wmsa.configuration.server.Context; -import nu.marginalia.wmsa.configuration.server.Initialization; -import nu.marginalia.wmsa.configuration.server.MetricsServer; -import nu.marginalia.wmsa.configuration.server.Service; -import nu.marginalia.wmsa.renderer.client.RendererClient; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiIndexReq; -import nu.marginalia.wmsa.renderer.request.smhi.RenderSmhiPrognosReq; -import nu.marginalia.wmsa.smhi.model.Plats; -import nu.marginalia.wmsa.smhi.model.PrognosData; -import nu.marginalia.wmsa.smhi.scraper.crawler.SmhiCrawler; -import nu.marginalia.wmsa.smhi.scraper.crawler.entity.SmhiEntityStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Spark; - -import java.util.Comparator; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -public class SmhiScraperService extends Service { - - private final SmhiCrawler crawler; - private final SmhiEntityStore entityStore; - private final RendererClient rendererClient; - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Initialization initialization; - @Inject - public SmhiScraperService(@Named("service-host") String ip, - @Named("service-port") Integer port, - SmhiCrawler crawler, - SmhiEntityStore entityStore, - RendererClient rendererClient, - Initialization initialization, - MetricsServer metricsServer) { - super(ip, port, initialization, metricsServer); - this.crawler = crawler; - this.entityStore = entityStore; - this.rendererClient = rendererClient; - this.initialization = initialization; - - Spark.awaitInitialization(); - - Schedulers.newThread().scheduleDirect(this::start); - } - - private void start() { - initialization.waitReady(); - rendererClient.waitReady(); - - entityStore.platser.debounce(6, TimeUnit.SECONDS) - .subscribe(this::updateIndex); - entityStore.prognosdata.subscribe(this::updatePrognos); - - crawler.start(); - } - - private void updatePrognos(PrognosData prognosData) { - rendererClient - .render(Context.internal(), new RenderSmhiPrognosReq(prognosData)) - .timeout(30, TimeUnit.SECONDS) - .blockingSubscribe(); - } - - private void updateIndex(Plats unused) { - var platser = entityStore.platser().stream() - .sorted(Comparator.comparing(plats -> plats.namn)) - .collect(Collectors.toList()); - - rendererClient - .render(Context.internal(), new RenderSmhiIndexReq(platser)) - .timeout(30, TimeUnit.SECONDS) - .blockingSubscribe(); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Parameter.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Parameter.java deleted file mode 100644 index 012e9c24..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Parameter.java +++ /dev/null @@ -1,9 +0,0 @@ -package nu.marginalia.wmsa.smhi.model; - -public class Parameter { - public String name; - public String levelType; - public String level; - public String unit; - public String[] values; -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Plats.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Plats.java deleted file mode 100644 index 7ae39675..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Plats.java +++ /dev/null @@ -1,42 +0,0 @@ -package nu.marginalia.wmsa.smhi.model; - -import lombok.Getter; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -@Getter -public class Plats { - public final String namn; - public final double latitud; - public final double longitud; - - public String getUrl() { - return namn.toLowerCase()+".html"; - } - - public Plats(String namn, String latitud, String longitud) { - this.namn = namn; - this.longitud = Double.parseDouble(longitud); - this.latitud = Double.parseDouble(latitud); - } - - public String toString() { - return String.format("Plats[%s %s %s]", namn, longitud, latitud); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - - if (o == null || getClass() != o.getClass()) return false; - - Plats plats = (Plats) o; - - return new EqualsBuilder().append(latitud, plats.latitud).append(longitud, plats.longitud).append(namn, plats.namn).isEquals(); - } - - @Override - public int hashCode() { - return new HashCodeBuilder(17, 37).append(namn).append(latitud).append(longitud).toHashCode(); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Platser.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Platser.java deleted file mode 100644 index c0f7c15f..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Platser.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.wmsa.smhi.model; - - -import java.util.List; - -public class Platser { - private final List platser; - - public Platser(List platser) { - this.platser = platser; - } - - public List getPlatser() { - return platser; - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/PrognosData.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/PrognosData.java deleted file mode 100644 index 1bb1be02..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/PrognosData.java +++ /dev/null @@ -1,41 +0,0 @@ -package nu.marginalia.wmsa.smhi.model; - -import nu.marginalia.wmsa.smhi.model.dyn.Dygnsdata; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -public class PrognosData { - - public final String crawlTime = LocalDateTime.now().toString(); - - public String approvedTime; - public String referenceTime; - public String expires; - - public Plats plats; - - public final List timeSeries = new ArrayList<>(); - - public String getBastFore() { - return LocalDateTime.parse(crawlTime).atZone(ZoneId.of("Europe/Stockholm")) - .plusHours(3) - .format(DateTimeFormatter.ISO_TIME); - } - public Plats getPlats() { - return plats; - } - - public List getTidpunkter() { - return timeSeries; - } - public List getDygn() { - return timeSeries.stream().map(Tidpunkt::getDate).distinct() - .map(datum -> new Dygnsdata(datum, this)) - .collect(Collectors.toList()); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Tidpunkt.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Tidpunkt.java deleted file mode 100644 index a4616df2..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/Tidpunkt.java +++ /dev/null @@ -1,75 +0,0 @@ -package nu.marginalia.wmsa.smhi.model; - -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.ArrayList; -import java.util.List; - -public class Tidpunkt { - - private static final ZoneId serverZoneId = ZoneId.of("GMT"); - private static final ZoneId localZoneId = ZoneId.of("Europe/Stockholm"); - private static final DateTimeFormatter timeFormatter = (new DateTimeFormatterBuilder()) - .appendValue(ChronoField.HOUR_OF_DAY, 2) - .appendLiteral(':') - .appendValue(ChronoField.MINUTE_OF_HOUR, 2) - .toFormatter(); - - public String validTime; - - public final List parameters = new ArrayList<>(); - - - private String getParam(String name) { - var data = parameters.stream().filter(p -> name.equals(p.name)).map(p->p.values).findFirst().orElseGet(() -> new String[0]); - if (data.length > 0) { - return data[0]; - } - return null; - } - public String getDate() { - return ZonedDateTime.parse(validTime).toLocalDateTime().atZone(serverZoneId).toOffsetDateTime().atZoneSameInstant(localZoneId).format(DateTimeFormatter.ISO_LOCAL_DATE); - } - - public String getTime() { - return ZonedDateTime.parse(validTime).toLocalDateTime().atZone(serverZoneId).toOffsetDateTime().atZoneSameInstant(localZoneId).format(timeFormatter); - } - - public String getTemp() { - return getParam("t"); - } - public String getMoln() { - return getParam("tcc_mean"); - } - public String getVind() { - return getParam("ws"); - } - public String getByvind() { - return getParam("gust"); - } - public String getNederbord() { - return getParam("pmedian"); - } - public String getNederbordTyp() { - switch(getParam("pcat")) { - case "1": return "S"; - case "2": return "SB"; - case "3": return "R"; - case "4": return "D"; - case "5": return "UKR"; - case "6": return "UKD"; - default: - return ""; - - } - } - public String getVindRiktning() { - return getParam("wd"); - } - public String toString() { - return String.format("Tidpunkt[%s %s]", validTime, getTemp()); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/dyn/Dygnsdata.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/dyn/Dygnsdata.java deleted file mode 100644 index 05f2246a..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/dyn/Dygnsdata.java +++ /dev/null @@ -1,40 +0,0 @@ -package nu.marginalia.wmsa.smhi.model.dyn; - -import nu.marginalia.wmsa.smhi.model.PrognosData; -import nu.marginalia.wmsa.smhi.model.Tidpunkt; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.stream.Collectors; - -public class Dygnsdata { - public final String date; - private final PrognosData data; - - public Dygnsdata(String date, PrognosData data) { - this.date = date; - this.data = data; - } - - public String getDate() { - return date; - } - public List getData() { - String d = getDate(); - return data.timeSeries.stream().filter(p -> d.equals(p.getDate())).collect(Collectors.toList()); - } - - public String getVeckodag() { - switch (LocalDate.parse(date, DateTimeFormatter.ISO_LOCAL_DATE).getDayOfWeek()) { - case MONDAY: return "Måndag"; - case TUESDAY: return "Tisdag"; - case WEDNESDAY: return "Onsdag"; - case THURSDAY: return "Torsdag"; - case FRIDAY: return "Fredag"; - case SATURDAY: return "Lördag"; - case SUNDAY: return "Söndag"; - } - return "Annandag"; - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlats.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlats.java deleted file mode 100644 index 5e3f3a19..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlats.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.wmsa.smhi.model.index; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.wmsa.smhi.model.Plats; - -import java.util.List; - -@Getter @AllArgsConstructor -public class IndexPlats { - String nyckel; - List platser; -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlatser.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlatser.java deleted file mode 100644 index 7300bcc5..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/model/index/IndexPlatser.java +++ /dev/null @@ -1,28 +0,0 @@ -package nu.marginalia.wmsa.smhi.model.index; - -import lombok.Getter; -import nu.marginalia.wmsa.smhi.model.Plats; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -@Getter -public class IndexPlatser { - final List platserPerNyckel = new ArrayList<>(); - - public IndexPlatser(List platser) { - var platsMap = kategoriseraEfterNyckel(platser); - - platsMap.keySet().stream().sorted() - .forEach(p -> platserPerNyckel.add(new IndexPlats(p, platsMap.get(p)))); - } - - private Map> kategoriseraEfterNyckel(List platser) { - return platser.stream().collect( - Collectors.groupingBy(p -> - p.namn.substring(0, 1) - .toUpperCase())); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/PlatsReader.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/PlatsReader.java deleted file mode 100644 index 3ea0d8cc..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/PlatsReader.java +++ /dev/null @@ -1,44 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.opencsv.CSVReader; -import nu.marginalia.wmsa.smhi.model.Plats; - -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -@Singleton -public class PlatsReader { - private final String fileName; - - @Inject - public PlatsReader(@Named("plats-csv-file") String fileName) { - this.fileName = fileName; - } - - public List readPlatser() throws Exception { - List platser = new ArrayList<>(); - - var resource = Objects.requireNonNull(ClassLoader.getSystemResourceAsStream(fileName), - "Kunde inte ladda " + fileName); - try (var reader = new CSVReader(new InputStreamReader(resource, StandardCharsets.UTF_8))) { - for (;;) { - String[] strings = reader.readNext(); - if (strings == null) { - return platser; - } - platser.add(skapaPlats(strings)); - } - } - - } - - private Plats skapaPlats(String[] strings) { - return new Plats(strings[0], strings[1], strings[2]); - } -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperMain.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperMain.java deleted file mode 100644 index d3edbf14..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperMain.java +++ /dev/null @@ -1,32 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.wmsa.configuration.MainClass; -import nu.marginalia.wmsa.configuration.ServiceDescriptor; -import nu.marginalia.wmsa.configuration.module.ConfigurationModule; -import nu.marginalia.wmsa.configuration.server.Initialization; -import nu.marginalia.wmsa.smhi.SmhiScraperService; - -import java.io.IOException; - -public class SmhiScraperMain extends MainClass { - private final SmhiScraperService service; - - @Inject - public SmhiScraperMain(SmhiScraperService service) { - this.service = service; - } - - public static void main(String... args) { - init(ServiceDescriptor.SMHI_SCRAPER, args); - - Injector injector = Guice.createInjector( - new SmhiScraperModule(), - new ConfigurationModule()); - injector.getInstance(SmhiScraperMain.class); - injector.getInstance(Initialization.class).setReady(); - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperModule.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperModule.java deleted file mode 100644 index ffb1793a..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/SmhiScraperModule.java +++ /dev/null @@ -1,12 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper; - -import com.google.inject.AbstractModule; -import com.google.inject.name.Names; - -public class SmhiScraperModule extends AbstractModule { - public void configure() { - bind(String.class).annotatedWith(Names.named("plats-csv-file")).toInstance("data/smhi/stader.csv"); - bind(String.class).annotatedWith(Names.named("smhi-user-agent")).toInstance("kontakt@marginalia.nu"); - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApi.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApi.java deleted file mode 100644 index 9880e317..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApi.java +++ /dev/null @@ -1,88 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper.crawler; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import nu.marginalia.wmsa.smhi.model.Plats; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.Locale; - -@Singleton -public class SmhiBackendApi { - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private final String server = "https://opendata-download-metfcst.smhi.se/api"; - private final PoolingHttpClientConnectionManager connectionManager; - private final String userAgent; - - @Inject - public SmhiBackendApi(@Named("smhi-user-agent") String userAgent) { - this.userAgent = userAgent; - - connectionManager = new PoolingHttpClientConnectionManager(); - connectionManager.setMaxTotal(200); - connectionManager.setDefaultMaxPerRoute(20); - HttpHost host = new HttpHost("https://opendata-download-metfcst.smhi.se"); - connectionManager.setMaxPerRoute(new HttpRoute(host), 50); - } - - public SmhiApiRespons hamtaData(Plats plats) throws Exception { - var client = HttpClients.custom() - .setConnectionManager(connectionManager) - .build(); - - String url = String.format(Locale.US, "%s/category/pmp3g/version/2/geotype/point/lon/%f/lat/%f/data.json", - server, plats.longitud, plats.latitud); - - Thread.sleep(100); - - logger.info("Fetching {} - {}", plats, url); - - HttpGet get = new HttpGet(url); - get.addHeader("User-Agent", userAgent); - - try (var rsp = client.execute(get)) { - var entity = rsp.getEntity(); - String content = new String(entity.getContent().readAllBytes()); - int statusCode = rsp.getStatusLine().getStatusCode(); - - var expires = - Arrays.stream(rsp.getHeaders("Expires")) - .map(Header::getValue) - .map(DateTimeFormatter.RFC_1123_DATE_TIME::parse) - .map(LocalDateTime::from) - .findFirst().map(Object::toString).orElse(""); - - - if (statusCode == 200) { - return new SmhiApiRespons(content, expires, plats); - } - throw new IllegalStateException("Fel i backend " + statusCode + " " + content); - } - - } - -} - -class SmhiApiRespons { - public final String jsonContent; - public final String expiryDate; - public final Plats plats; - - SmhiApiRespons(String jsonContent, String expiryDate, Plats plats) { - this.jsonContent = jsonContent; - this.expiryDate = expiryDate; - this.plats = plats; - } -} \ No newline at end of file diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiCrawler.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiCrawler.java deleted file mode 100644 index c97da68d..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiCrawler.java +++ /dev/null @@ -1,106 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper.crawler; - -import com.google.gson.*; -import com.google.inject.Inject; -import io.reactivex.rxjava3.core.Maybe; -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.disposables.Disposable; -import io.reactivex.rxjava3.schedulers.Schedulers; -import lombok.SneakyThrows; -import nu.marginalia.wmsa.smhi.model.Plats; -import nu.marginalia.wmsa.smhi.model.PrognosData; -import nu.marginalia.wmsa.smhi.scraper.PlatsReader; -import nu.marginalia.wmsa.smhi.scraper.crawler.entity.SmhiEntityStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Type; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -public class SmhiCrawler { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private final Gson gson; - private final SmhiBackendApi api; - private final SmhiEntityStore store; - private final List platser; - private Disposable job; - - @Inject @SneakyThrows - public SmhiCrawler(SmhiBackendApi backendApi, SmhiEntityStore store, PlatsReader platsReader) { - this.api = backendApi; - this.store = store; - this.platser = platsReader.readPlatser(); - - class LocalDateAdapter implements JsonDeserializer { - @Override - public LocalDateTime deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { - return LocalDateTime - .parse(json.getAsString(), DateTimeFormatter.ISO_ZONED_DATE_TIME); - } - } - - gson = new GsonBuilder() - .registerTypeAdapter(LocalDateTime.class, new LocalDateAdapter()) - .create(); - } - - public void start() { - job = Observable - .fromIterable(new ArrayList<>(platser)) - .subscribeOn(Schedulers.io()) - .filter(this::isNeedsUpdate) - .take(5) - .flatMapMaybe(this::hamtaData) - .repeatWhen(this::repeatDelay) - .doOnError(this::handleError) - .subscribe(store::offer); - } - public void stop() { - Optional.ofNullable(job).ifPresent(Disposable::dispose); - } - - private Observable repeatDelay(Observable completed) { - return completed.delay(1, TimeUnit.SECONDS); - } - - protected void handleError(Throwable throwable) { - logger.error("Caught error", throwable); - } - - public Maybe hamtaData(Plats plats) { - try { - var data = api.hamtaData(plats); - - PrognosData model = gson.fromJson(data.jsonContent, PrognosData.class); - - model.expires = data.expiryDate; - model.plats = plats; - - return Maybe.just(model); - } - catch (Exception ex) { - logger.error("Failed to fetch data", ex); - return Maybe.empty(); - } - } - - - boolean isNeedsUpdate(Plats plats) { - var prognos = store.prognos(plats); - - if (null == prognos) { - return true; - } - - LocalDateTime crawlTime = LocalDateTime.parse(prognos.crawlTime); - return crawlTime.plusHours(1).isBefore(LocalDateTime.now()); - } - -} diff --git a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/entity/SmhiEntityStore.java b/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/entity/SmhiEntityStore.java deleted file mode 100644 index d2f608aa..00000000 --- a/marginalia_nu/src/main/java/nu/marginalia/wmsa/smhi/scraper/crawler/entity/SmhiEntityStore.java +++ /dev/null @@ -1,62 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper.crawler.entity; - -import com.google.inject.Singleton; -import io.reactivex.rxjava3.subjects.PublishSubject; -import nu.marginalia.wmsa.smhi.model.Plats; -import nu.marginalia.wmsa.smhi.model.PrognosData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -@Singleton -public class SmhiEntityStore { - private final ReadWriteLock rwl = new ReentrantReadWriteLock(); - private final Map data = new HashMap<>(); - - public final PublishSubject platser = PublishSubject.create(); - public final PublishSubject prognosdata = PublishSubject.create(); - Logger logger = LoggerFactory.getLogger(getClass()); - public boolean offer(PrognosData modell) { - Lock lock = this.rwl.writeLock(); - try { - lock.lock(); - if (data.put(modell.plats, modell) == null) { - platser.onNext(modell.plats); - } - prognosdata.onNext(modell); - } - finally { - lock.unlock(); - } - return true; - } - - public List platser() { - Lock lock = this.rwl.readLock(); - try { - lock.lock(); - return new ArrayList<>(data.keySet()); - } - finally { - lock.unlock(); - } - } - - public PrognosData prognos(Plats plats) { - Lock lock = this.rwl.readLock(); - try { - lock.lock(); - return data.get(plats); - } - finally { - lock.unlock(); - } - } -} diff --git a/marginalia_nu/src/test/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApiTest.java b/marginalia_nu/src/test/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApiTest.java deleted file mode 100644 index d8b252f9..00000000 --- a/marginalia_nu/src/test/java/nu/marginalia/wmsa/smhi/scraper/crawler/SmhiBackendApiTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.wmsa.smhi.scraper.crawler; - -import nu.marginalia.wmsa.smhi.model.Plats; -import org.junit.jupiter.api.Test; - -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; - -class SmhiBackendApiTest { - - @Test - void hamtaData() throws Exception { - var api = new SmhiBackendApi("nu.marginalia"); - - - System.out.println(api.hamtaData(new Plats("Ystad", "55.42966", "13.82041")) - .jsonContent - ); - } - - @Test - public void testDatum() { - System.out.println(LocalDateTime.parse("2021-05-29T14:06:48Z", - DateTimeFormatter.ISO_ZONED_DATE_TIME) - .atZone(ZoneId.of("GMT")) - .toOffsetDateTime() - .atZoneSameInstant(ZoneId.of("Europe/Stockholm")) - ); - } -} \ No newline at end of file