From 2eb972dea1205bbc2d7dc4e3b3978bc85f860269 Mon Sep 17 00:00:00 2001 From: Viktor Lofgren Date: Fri, 17 Mar 2023 16:03:11 +0100 Subject: [PATCH] Remove unrelated code, break tools into their own directory. --- .../marginalia/model/idx/DocumentFlags.java | 4 +- .../keyword/DocumentKeywordExtractor.java | 2 +- .../marginalia/keyword/KeywordExtractor.java | 2 +- .../model/IndexJournalEntryBuilder.java | 5 - .../features-search/query-parser/build.gradle | 4 +- .../language}/EnglishDictionary.java | 2 +- .../nu/marginalia/ngrams}/DenseBitMap.java | 2 +- .../marginalia/ngrams}/NGramBloomFilter.java | 2 +- .../query_parser/QueryVariants.java | 4 +- .../marginalia/ngrams}/DenseBitMapTest.java | 2 +- .../query_parser/BodyQueryParserTest.java | 4 +- .../query_parser/QueryVariantsTest.java | 4 +- .../factors/TermCoherenceFactorTest.java | 4 +- .../nu/marginalia/language/WordPatterns.java | 54 +--- .../libraries/ngram-bloom-filter/build.gradle | 42 --- .../term-frequency-dict/build.gradle | 1 + code/libraries/term-frequency-dict/readme.md | 12 + .../TermFrequencyDict.java | 157 +++------- .../process-models/converting-model/readme.md | 47 ++- code/process-models/crawling-model/readme.md | 1 + .../converting}/language/LanguageFilter.java | 2 +- .../converting/model/ProcessedDocument.java | 2 +- .../AbstractDocumentProcessorPlugin.java | 2 +- .../converting/ConvertingIntegrationTest.java | 21 +- .../language}/LanguageFilterTest.java | 3 +- .../test/resources/memex-marginalia/readme.md | 3 + .../crawl-job-extractor-process/readme.md | 4 - code/processes/readme.md | 14 +- code/readme.md | 6 +- .../services-core/search-service/build.gradle | 1 - .../marginalia/search/query/QueryFactory.java | 4 +- .../search/query/QueryFactoryTest.java | 4 +- .../crawl-job-extractor}/build.gradle | 2 +- code/tools/crawl-job-extractor/readme.md | 6 + .../crawl/CrawlJobDomainExtractor.java | 0 .../crawl/CrawlJobExtractorMain.java | 0 .../marginalia/crawl/CrawlJobSpecWriter.java | 0 .../crawl/CrawlJobSpecWriterTest.java | 0 .../term-frequency-extractor/build.gradle | 62 ++++ code/tools/term-frequency-extractor/readme.md | 16 + .../tools/TermFrequencyExtractor.java | 114 +++++++ other/memex/build.gradle | 242 --------------- other/memex/lombok.config | 2 - .../memex/MemexServiceDescriptors.java | 15 - .../memex/auth/AuthConfigurationModule.java | 14 - .../nu/marginalia/memex/auth/AuthMain.java | 27 -- .../nu/marginalia/memex/auth/AuthService.java | 118 ------- .../memex/auth/client/AuthClient.java | 45 --- .../memex/auth/model/LoginFormModel.java | 10 - .../marginalia/memex/gemini/BadBotList.java | 43 --- .../gemini/GeminiConfigurationModule.java | 17 - .../memex/gemini/GeminiService.java | 7 - .../memex/gemini/GeminiServiceDummy.java | 10 - .../memex/gemini/GeminiServiceImpl.java | 164 ---------- .../memex/gemini/client/GeminiClient.java | 130 -------- .../marginalia/memex/gemini/gmi/Gemtext.java | 53 ---- .../memex/gemini/gmi/GemtextDatabase.java | 71 ----- .../memex/gemini/gmi/GemtextDocument.java | 163 ---------- .../gemini/gmi/line/AbstractGemtextLine.java | 18 -- .../memex/gemini/gmi/line/GemtextAside.java | 21 -- .../memex/gemini/gmi/line/GemtextHeading.java | 32 -- .../gemini/gmi/line/GemtextLineVisitor.java | 18 -- .../gmi/line/GemtextLineVisitorAdapter.java | 53 ---- .../memex/gemini/gmi/line/GemtextLink.java | 33 -- .../memex/gemini/gmi/line/GemtextList.java | 23 -- .../memex/gemini/gmi/line/GemtextPragma.java | 21 -- .../gemini/gmi/line/GemtextPreformat.java | 23 -- .../memex/gemini/gmi/line/GemtextQuote.java | 23 -- .../memex/gemini/gmi/line/GemtextTask.java | 42 --- .../memex/gemini/gmi/line/GemtextText.java | 21 -- .../gemini/gmi/line/GemtextTextLiteral.java | 23 -- .../gemini/gmi/parser/GemtextAsideParser.java | 20 -- .../gmi/parser/GemtextHeadingParser.java | 26 -- .../gemini/gmi/parser/GemtextLinkParser.java | 42 --- .../gemini/gmi/parser/GemtextListParser.java | 17 - .../gemini/gmi/parser/GemtextParser.java | 135 -------- .../gmi/parser/GemtextPragmaParser.java | 26 -- .../gemini/gmi/parser/GemtextQuoteParser.java | 17 - .../gemini/gmi/parser/GemtextTaskParser.java | 31 -- .../gemini/gmi/renderer/GemtextRenderer.java | 91 ------ .../gmi/renderer/GemtextRendererFactory.java | 227 -------------- .../memex/gemini/io/GeminiConnection.java | 185 ----------- .../memex/gemini/io/GeminiSSLSetUp.java | 49 --- .../memex/gemini/io/GeminiStatusCode.java | 11 - .../memex/gemini/io/GeminiUserException.java | 8 - .../gemini/plugins/BareStaticPagePlugin.java | 52 ---- .../memex/gemini/plugins/FileType.java | 58 ---- .../memex/gemini/plugins/Plugin.java | 19 -- .../memex/gemini/plugins/SearchPlugin.java | 78 ----- .../java/nu/marginalia/memex/memex/Memex.java | 244 --------------- .../memex/memex/MemexConfigurationModule.java | 87 ------ .../nu/marginalia/memex/memex/MemexData.java | 150 --------- .../nu/marginalia/memex/memex/MemexLinks.java | 54 ---- .../marginalia/memex/memex/MemexLoader.java | 265 ---------------- .../nu/marginalia/memex/memex/MemexMain.java | 31 -- .../marginalia/memex/memex/MemexService.java | 292 ------------------ .../memex/memex/change/GemtextAppend.java | 70 ----- .../memex/memex/change/GemtextCreate.java | 19 -- .../memex/change/GemtextCreateOrMutate.java | 26 -- .../memex/memex/change/GemtextMutation.java | 18 -- .../memex/memex/change/GemtextPrepend.java | 63 ---- .../memex/memex/change/GemtextReplace.java | 65 ---- .../GemtextTombstoneUpdateCaclulator.java | 48 --- .../GemtextDocumentUpdateCalculator.java | 109 ------- .../change/update/GemtextTaskExtractor.java | 31 -- .../change/update/GemtextTasksRewrite.java | 100 ------ .../memex/memex/model/GemtextSection.java | 11 - .../memex/model/GemtextSectionAction.java | 6 - .../memex/memex/model/MemexExternalUrl.java | 18 -- .../memex/memex/model/MemexImage.java | 13 - .../memex/memex/model/MemexIndexTask.java | 13 - .../memex/memex/model/MemexLink.java | 26 -- .../memex/memex/model/MemexNode.java | 40 --- .../memex/memex/model/MemexNodeHeadingId.java | 74 ----- .../memex/memex/model/MemexNodeTaskId.java | 66 ---- .../memex/memex/model/MemexNodeType.java | 14 - .../memex/memex/model/MemexNodeUrl.java | 98 ------ .../memex/memex/model/MemexTaskState.java | 30 -- .../memex/memex/model/MemexTaskTags.java | 40 --- .../memex/memex/model/MemexUrl.java | 13 - .../memex/memex/model/fs/MemexDirectory.java | 30 -- .../memex/memex/model/fs/MemexFileSystem.java | 88 ------ .../render/MemexRenderCreateFormModel.java | 32 -- .../render/MemexRenderUpdateFormModel.java | 19 -- .../render/MemexRenderUploadFormModel.java | 32 -- .../render/MemexRendererDeleteFormModel.java | 19 -- .../model/render/MemexRendererImageModel.java | 36 --- .../model/render/MemexRendererIndexModel.java | 71 ----- .../render/MemexRendererRenameFormModel.java | 19 -- .../render/MemexRendererTombstoneModel.java | 16 - .../model/render/MemexRendererViewModel.java | 24 -- .../model/render/MemexRendererableDirect.java | 7 - .../memex/renderer/MemexGmiRenderer.java | 228 -------------- .../memex/renderer/MemexHtmlRenderer.java | 199 ------------ .../memex/renderer/MemexRendererers.java | 22 -- .../system/MemexFileSystemModifiedTimes.java | 23 -- .../memex/system/MemexFileSystemMonitor.java | 116 ------- .../memex/memex/system/MemexFileWriter.java | 120 ------- .../memex/system/MemexSourceFileSystem.java | 87 ------ .../memex/memex/system/git/MemexGitRepo.java | 15 - .../memex/system/git/MemexGitRepoDummy.java | 36 --- .../memex/system/git/MemexGitRepoImpl.java | 141 --------- .../memex/renderer/MustacheRenderer.java | 164 ---------- .../memex/renderer/RendererFactory.java | 13 - .../util/dithering/FloydSteinbergDither.java | 243 --------------- .../memex/util/dithering/Palettes.java | 29 -- .../java/nu/marginalia/util/FileSizeUtil.java | 18 -- .../main/resources/static/memex/ico/dir.png | Bin 174 -> 0 bytes .../main/resources/static/memex/ico/doc32.png | Bin 247 -> 0 bytes .../main/resources/static/memex/ico/file.png | Bin 141 -> 0 bytes .../resources/static/memex/ico/folder32.png | Bin 240 -> 0 bytes .../resources/static/memex/ico/folderup16.png | Bin 251 -> 0 bytes .../main/resources/static/memex/ico/nav16.png | Bin 301 -> 0 bytes .../main/resources/static/memex/ico/pic16.png | Bin 296 -> 0 bytes .../main/resources/static/memex/ico/pic32.png | Bin 472 -> 0 bytes .../main/resources/static/memex/ico/root.png | Bin 301 -> 0 bytes .../resources/static/memex/ico/shiba16.png | Bin 316 -> 0 bytes .../resources/static/memex/ico/world16.png | Bin 343 -> 0 bytes .../main/resources/static/memex/style-new.css | 241 --------------- .../main/resources/templates/auth/login.hdb | 36 --- .../templates/memex/memex-create-form.hdb | 23 -- .../templates/memex/memex-delete-form.hdb | 23 -- .../resources/templates/memex/memex-image.hdb | 24 -- .../templates/memex/memex-index-feed.hdb | 20 -- .../resources/templates/memex/memex-index.hdb | 34 -- .../templates/memex/memex-rename-form.hdb | 23 -- .../templates/memex/memex-tombstone.hdb | 17 - .../templates/memex/memex-update-form.hdb | 20 -- .../templates/memex/memex-upload-form.hdb | 26 -- .../resources/templates/memex/memex-view.hdb | 36 --- .../memex/partial/memex-backlinks-inline.hdb | 6 - .../memex/partial/memex-backlinks.hdb | 9 - .../memex/partial/memex-directories.hdb | 9 - .../memex/partial/memex-documents-inline.hdb | 10 - .../memex/partial/memex-documents.hdb | 10 - .../templates/memex/partial/memex-footer.hdb | 4 - .../templates/memex/partial/memex-head.hdb | 9 - .../templates/memex/partial/memex-images.hdb | 6 - .../memex/partial/memex-task-listing.hdb | 10 - .../templates/memex/partial/memex-topbar.hdb | 13 - .../marginalia/gmi/GemtextDatabaseTest.java | 31 -- .../marginalia/gmi/GemtextDocumentTest.java | 90 ------ .../gmi/parser/GemtextTaskParserTest.java | 19 -- .../dithering/FloydSteinbergDitherTest.java | 34 -- .../memex/memex/MemexFileWriterTest.java | 43 --- .../nu/marginalia/memex/memex/MemexTest.java | 11 - .../memex/memex/change/GemtextChangeTest.java | 275 ----------------- .../memex/change/GemtextTaskUpdateTest.java | 227 -------------- .../GemtextTombstoneUpdateCaclulatorTest.java | 107 ------- .../memex/model/MemexNodeHeadingIdTest.java | 33 -- .../nu/marginalia/util/test/TestUtil.java | 50 --- other/readme.md | 5 - other/wmsa_old/build.gradle | 112 ------- .../java/nu/marginalia/wmsa/WmsaHome.java | 56 ---- .../wmsa/podcasts/PodcastFetcher.java | 112 ------- .../wmsa/podcasts/PodcastScraperMain.java | 31 -- .../wmsa/podcasts/PodcastScraperService.java | 78 ----- .../wmsa/podcasts/model/Podcast.java | 16 - .../wmsa/podcasts/model/PodcastEpisode.java | 16 - .../wmsa/podcasts/model/PodcastListing.java | 16 - .../wmsa/podcasts/model/PodcastMetadata.java | 17 - .../podcasts/model/PodcastNewEpisodes.java | 11 - .../wmsa/renderer/PodcastRendererService.java | 128 -------- .../wmsa/renderer/RendererMain.java | 29 -- .../wmsa/renderer/RendererModule.java | 8 - .../wmsa/renderer/RendererService.java | 35 --- .../wmsa/renderer/ServerStatusModel.java | 10 - .../wmsa/renderer/WmsaServiceDescriptors.java | 17 - .../wmsa/renderer/client/RendererClient.java | 53 ---- .../renderer/mustache/MustacheRenderer.java | 61 ---- .../renderer/mustache/RendererFactory.java | 13 - .../resource_store/ResourceEntityStore.java | 247 --------------- .../resource_store/ResourceStoreClient.java | 49 --- .../resource_store/ResourceStoreMain.java | 30 -- .../resource_store/ResourceStoreModule.java | 15 - .../resource_store/ResourceStoreService.java | 158 ---------- .../model/RenderedResource.java | 46 --- .../main/resources/static/podcast/style.css | 3 - .../resources/templates/podcast/episode.hdb | 35 --- .../resources/templates/podcast/listing.hdb | 33 -- .../main/resources/templates/podcast/new.hdb | 34 -- .../resources/templates/podcast/podcast.hdb | 38 --- .../ResourceStoreServiceTest.java | 123 -------- settings.gradle | 9 +- 224 files changed, 377 insertions(+), 9972 deletions(-) rename code/{libraries/language-processing/src/main/java/nu/marginalia/language/statistics => features-search/query-parser/src/main/java/nu/marginalia/language}/EnglishDictionary.java (99%) rename code/{libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter => features-search/query-parser/src/main/java/nu/marginalia/ngrams}/DenseBitMap.java (97%) rename code/{libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter => features-search/query-parser/src/main/java/nu/marginalia/ngrams}/NGramBloomFilter.java (97%) rename code/{libraries/ngram-bloom-filter/src/test/java/nu/marginalia/ngram_bloom_filter => features-search/query-parser/src/test/java/nu/marginalia/ngrams}/DenseBitMapTest.java (97%) delete mode 100644 code/libraries/ngram-bloom-filter/build.gradle create mode 100644 code/libraries/term-frequency-dict/readme.md rename code/{libraries/language-processing/src/main/java/nu/marginalia => processes/converting-process/src/main/java/nu/marginalia/converting}/language/LanguageFilter.java (98%) rename code/processes/{crawling-process/src/test/java/nu/marginalia/crawling => converting-process/src/test/java/nu/marginalia/converting/language}/LanguageFilterTest.java (93%) create mode 100644 code/processes/converting-process/src/test/resources/memex-marginalia/readme.md delete mode 100644 code/processes/crawl-job-extractor-process/readme.md rename code/{processes/crawl-job-extractor-process => tools/crawl-job-extractor}/build.gradle (96%) create mode 100644 code/tools/crawl-job-extractor/readme.md rename code/{processes/crawl-job-extractor-process => tools/crawl-job-extractor}/src/main/java/nu/marginalia/crawl/CrawlJobDomainExtractor.java (100%) rename code/{processes/crawl-job-extractor-process => tools/crawl-job-extractor}/src/main/java/nu/marginalia/crawl/CrawlJobExtractorMain.java (100%) rename code/{processes/crawl-job-extractor-process => tools/crawl-job-extractor}/src/main/java/nu/marginalia/crawl/CrawlJobSpecWriter.java (100%) rename code/{processes/crawl-job-extractor-process => tools/crawl-job-extractor}/src/test/java/nu/marginalia/crawl/CrawlJobSpecWriterTest.java (100%) create mode 100644 code/tools/term-frequency-extractor/build.gradle create mode 100644 code/tools/term-frequency-extractor/readme.md create mode 100644 code/tools/term-frequency-extractor/src/main/java/nu/marginalia/tools/TermFrequencyExtractor.java delete mode 100644 other/memex/build.gradle delete mode 100644 other/memex/lombok.config delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/MemexServiceDescriptors.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/auth/AuthConfigurationModule.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/auth/AuthMain.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/auth/AuthService.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/auth/client/AuthClient.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/auth/model/LoginFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/BadBotList.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiConfigurationModule.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiService.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceDummy.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceImpl.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/client/GeminiClient.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/Gemtext.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDatabase.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDocument.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/AbstractGemtextLine.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextAside.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextHeading.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitor.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitorAdapter.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLink.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextList.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPragma.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPreformat.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextQuote.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTask.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextText.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTextLiteral.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextAsideParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextHeadingParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextLinkParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextListParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextPragmaParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextQuoteParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextTaskParser.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRenderer.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRendererFactory.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiConnection.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiSSLSetUp.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiStatusCode.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiUserException.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/BareStaticPagePlugin.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/FileType.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/Plugin.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/SearchPlugin.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/Memex.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexConfigurationModule.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexData.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexLinks.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexLoader.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexMain.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/MemexService.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextAppend.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreate.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreateOrMutate.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextMutation.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextPrepend.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextReplace.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulator.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextDocumentUpdateCalculator.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTaskExtractor.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTasksRewrite.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSection.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSectionAction.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexExternalUrl.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexImage.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexIndexTask.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexLink.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNode.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeHeadingId.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeTaskId.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeType.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeUrl.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskState.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskTags.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexUrl.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexDirectory.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexFileSystem.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderCreateFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUpdateFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUploadFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererDeleteFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererImageModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererIndexModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererRenameFormModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererTombstoneModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererViewModel.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererableDirect.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexGmiRenderer.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexHtmlRenderer.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexRendererers.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemModifiedTimes.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemMonitor.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileWriter.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexSourceFileSystem.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepo.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoDummy.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoImpl.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/renderer/MustacheRenderer.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/renderer/RendererFactory.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/util/dithering/FloydSteinbergDither.java delete mode 100644 other/memex/src/main/java/nu/marginalia/memex/util/dithering/Palettes.java delete mode 100644 other/memex/src/main/java/nu/marginalia/util/FileSizeUtil.java delete mode 100644 other/memex/src/main/resources/static/memex/ico/dir.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/doc32.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/file.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/folder32.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/folderup16.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/nav16.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/pic16.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/pic32.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/root.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/shiba16.png delete mode 100644 other/memex/src/main/resources/static/memex/ico/world16.png delete mode 100644 other/memex/src/main/resources/static/memex/style-new.css delete mode 100644 other/memex/src/main/resources/templates/auth/login.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-create-form.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-delete-form.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-image.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-index-feed.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-index.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-rename-form.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-tombstone.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-update-form.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-upload-form.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/memex-view.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-backlinks-inline.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-backlinks.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-directories.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-documents-inline.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-documents.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-footer.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-head.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-images.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-task-listing.hdb delete mode 100644 other/memex/src/main/resources/templates/memex/partial/memex-topbar.hdb delete mode 100644 other/memex/src/test/java/nu/marginalia/gmi/GemtextDatabaseTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/gmi/GemtextDocumentTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/gmi/parser/GemtextTaskParserTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/dithering/FloydSteinbergDitherTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/MemexFileWriterTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/MemexTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextChangeTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTaskUpdateTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulatorTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/memex/memex/model/MemexNodeHeadingIdTest.java delete mode 100644 other/memex/src/test/java/nu/marginalia/util/test/TestUtil.java delete mode 100644 other/readme.md delete mode 100644 other/wmsa_old/build.gradle delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/WmsaHome.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastFetcher.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperMain.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperService.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/Podcast.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastEpisode.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastListing.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastMetadata.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastNewEpisodes.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/PodcastRendererService.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererMain.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererModule.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/ServerStatusModel.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/WmsaServiceDescriptors.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/MustacheRenderer.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/RendererFactory.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceEntityStore.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreClient.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreMain.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreService.java delete mode 100644 other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/model/RenderedResource.java delete mode 100644 other/wmsa_old/src/main/resources/static/podcast/style.css delete mode 100644 other/wmsa_old/src/main/resources/templates/podcast/episode.hdb delete mode 100644 other/wmsa_old/src/main/resources/templates/podcast/listing.hdb delete mode 100644 other/wmsa_old/src/main/resources/templates/podcast/new.hdb delete mode 100644 other/wmsa_old/src/main/resources/templates/podcast/podcast.hdb delete mode 100644 other/wmsa_old/src/test/java/nu/marginalia/resource_store/ResourceStoreServiceTest.java diff --git a/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java b/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java index 7528a4eb..099d2da8 100644 --- a/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java +++ b/code/common/model/src/main/java/nu/marginalia/model/idx/DocumentFlags.java @@ -3,9 +3,7 @@ package nu.marginalia.model.idx; import java.util.EnumSet; public enum DocumentFlags { - /** Simple processing was done, this document should be de-prioritized as a search result */ - Simple, - + UnusedBit1, PlainText, UnusedBit2, UnusedBit3, diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java index 49d64002..af0f5073 100644 --- a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java +++ b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/DocumentKeywordExtractor.java @@ -67,7 +67,7 @@ public class DocumentKeywordExtractor { String flatWord = AsciiFlattener.flattenUnicode(word.word); - if (WordPatterns.hasWordQualities(flatWord)) { + if (!flatWord.isBlank()) { wordsBuilder.add(flatWord, metadata.getMetadataForWord(word.stemmed)); } } diff --git a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java index 6b3540f0..bdc87f29 100644 --- a/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java +++ b/code/features-convert/keyword-extraction/src/main/java/nu/marginalia/keyword/KeywordExtractor.java @@ -220,7 +220,7 @@ public class KeywordExtractor { } String word = sentence.constructWordFromSpan(w); - if (word.isBlank() || !WordPatterns.filter(word)) return false; + if (word.isBlank() || !WordPatterns.isNotJunkWord(word)) return false; if (sentence.posTags[w.start].equals("CC")) return false; if (sentence.posTags[w.end-1].equals("IN")) return false; if (sentence.posTags[w.end-1].equals("DT")) return false; diff --git a/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java index f5f1fc22..979af42d 100644 --- a/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java +++ b/code/features-index/index-journal/src/main/java/nu.marginalia.index/journal/model/IndexJournalEntryBuilder.java @@ -12,11 +12,6 @@ public class IndexJournalEntryBuilder { this.documentMeta = documentMeta; } - public IndexJournalEntryBuilder capacity(int size) { - items.ensureCapacity(size); - return this; - } - public IndexJournalEntryBuilder add(long wordId, long metadata) { items.add(wordId); diff --git a/code/features-search/query-parser/build.gradle b/code/features-search/query-parser/build.gradle index edf02741..c738a144 100644 --- a/code/features-search/query-parser/build.gradle +++ b/code/features-search/query-parser/build.gradle @@ -13,12 +13,13 @@ java { } dependencies { implementation project(':code:libraries:language-processing') - implementation project(':code:libraries:ngram-bloom-filter') implementation project(':code:libraries:term-frequency-dict') implementation project(':code:features-convert:keyword-extraction') implementation project(':code:common:config') implementation project(':code:common:model') + implementation project(':third-party:porterstemmer') + implementation libs.lombok annotationProcessor libs.lombok implementation libs.bundles.slf4j @@ -26,6 +27,7 @@ dependencies { implementation libs.bundles.handlebars implementation libs.trove + implementation libs.guice testImplementation libs.bundles.slf4j.test testImplementation libs.bundles.junit diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/statistics/EnglishDictionary.java b/code/features-search/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java similarity index 99% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/statistics/EnglishDictionary.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java index d96c0666..0afd3625 100644 --- a/code/libraries/language-processing/src/main/java/nu/marginalia/language/statistics/EnglishDictionary.java +++ b/code/features-search/query-parser/src/main/java/nu/marginalia/language/EnglishDictionary.java @@ -1,4 +1,4 @@ -package nu.marginalia.language.statistics; +package nu.marginalia.language; import com.google.inject.Inject; import nu.marginalia.term_frequency_dict.TermFrequencyDict; diff --git a/code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/DenseBitMap.java b/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java similarity index 97% rename from code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/DenseBitMap.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java index a69576cc..ca5666b1 100644 --- a/code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/DenseBitMap.java +++ b/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/DenseBitMap.java @@ -1,4 +1,4 @@ -package nu.marginalia.ngram_bloom_filter; +package nu.marginalia.ngrams; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/NGramBloomFilter.java b/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java similarity index 97% rename from code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/NGramBloomFilter.java rename to code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java index 85af1367..89a6d9cf 100644 --- a/code/libraries/ngram-bloom-filter/src/main/java/nu/marginalia/ngram_bloom_filter/NGramBloomFilter.java +++ b/code/features-search/query-parser/src/main/java/nu/marginalia/ngrams/NGramBloomFilter.java @@ -1,4 +1,4 @@ -package nu.marginalia.ngram_bloom_filter; +package nu.marginalia.ngrams; import ca.rmen.porterstemmer.PorterStemmer; import com.google.common.hash.HashFunction; diff --git a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java index 6acdaed4..eb4abd79 100644 --- a/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java +++ b/code/features-search/query-parser/src/main/java/nu/marginalia/query_parser/QueryVariants.java @@ -6,9 +6,9 @@ import lombok.Getter; import lombok.ToString; import nu.marginalia.LanguageModels; import nu.marginalia.keyword.KeywordExtractor; -import nu.marginalia.language.statistics.EnglishDictionary; +import nu.marginalia.language.EnglishDictionary; import nu.marginalia.language.sentence.SentenceExtractor; -import nu.marginalia.ngram_bloom_filter.NGramBloomFilter; +import nu.marginalia.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.language.model.DocumentSentence; import nu.marginalia.language.model.WordSpan; diff --git a/code/libraries/ngram-bloom-filter/src/test/java/nu/marginalia/ngram_bloom_filter/DenseBitMapTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java similarity index 97% rename from code/libraries/ngram-bloom-filter/src/test/java/nu/marginalia/ngram_bloom_filter/DenseBitMapTest.java rename to code/features-search/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java index 783b2ca9..d2db16b6 100644 --- a/code/libraries/ngram-bloom-filter/src/test/java/nu/marginalia/ngram_bloom_filter/DenseBitMapTest.java +++ b/code/features-search/query-parser/src/test/java/nu/marginalia/ngrams/DenseBitMapTest.java @@ -1,4 +1,4 @@ -package nu.marginalia.ngram_bloom_filter; +package nu.marginalia.ngrams; import org.junit.jupiter.api.Test; diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java index cd9a61eb..8cc38312 100644 --- a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java +++ b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/BodyQueryParserTest.java @@ -1,8 +1,8 @@ package nu.marginalia.query_parser; import nu.marginalia.LanguageModels; -import nu.marginalia.language.statistics.EnglishDictionary; -import nu.marginalia.ngram_bloom_filter.NGramBloomFilter; +import nu.marginalia.language.EnglishDictionary; +import nu.marginalia.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.query_parser.token.TokenType; import nu.marginalia.util.TestLanguageModels; diff --git a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java index e67a6940..d82976e9 100644 --- a/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java +++ b/code/features-search/query-parser/src/test/java/nu/marginalia/query_parser/QueryVariantsTest.java @@ -1,8 +1,8 @@ package nu.marginalia.query_parser; import nu.marginalia.LanguageModels; -import nu.marginalia.language.statistics.EnglishDictionary; -import nu.marginalia.ngram_bloom_filter.NGramBloomFilter; +import nu.marginalia.language.EnglishDictionary; +import nu.marginalia.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.util.TestLanguageModels; import nu.marginalia.language.sentence.SentenceExtractor; diff --git a/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java b/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java index c9d8dd00..685118d4 100644 --- a/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java +++ b/code/features-search/result-ranking/src/test/java/nu/marginalia/ranking/factors/TermCoherenceFactorTest.java @@ -41,7 +41,7 @@ class TermCoherenceFactorTest { assertEquals(0, termCoherenceFactor.calculate(allPositionsSet)); } - @Test + @Test @SuppressWarnings("unchecked") public void testLowPosMatches() { var allPositionsSet = createSet( List.of(0, 1, 2, 3), List.of(0, 1, 2, 3) @@ -53,7 +53,7 @@ class TermCoherenceFactorTest { assertEquals(1.0, termCoherenceFactor.bitPositionFactor(mask), 0.01); } - @Test + @Test @SuppressWarnings("unchecked") public void testHiPosMatches() { var allPositionsSet = createSet( List.of(28, 29, 30, 31), List.of(28, 29, 30, 31) diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java b/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java index 39483338..6758fdae 100644 --- a/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java +++ b/code/libraries/language-processing/src/main/java/nu/marginalia/language/WordPatterns.java @@ -8,8 +8,6 @@ import java.io.InputStreamReader; import java.util.HashSet; import java.util.Objects; import java.util.Set; -import java.util.function.Predicate; -import java.util.regex.Pattern; /** Regular expression patterns for deciding which words are eligible to be keywords. *

@@ -44,25 +42,17 @@ public class WordPatterns { } } - private static boolean hasMoreThanTwo(String s, char c, int max) { - int idx = 0; - for (int i = 0; i <= max; i++) { - idx = s.indexOf(c, idx+1); - if (idx < 0 || idx >= s.length() - 1) - return false; - } - return true; - } - - public static boolean filter(String word) { + /** Run checks on the word and exclude terms with too many special characters + */ + public static boolean isNotJunkWord(String word) { if (word.isBlank()) { return false; } - if (hasMoreThanTwo(word, '-', 4)) { + if (hasMoreThanN(word, '-', 4)) { return false; } - if (hasMoreThanTwo(word, '+', 2)) { + if (hasMoreThanN(word, '+', 2)) { return false; } if (word.startsWith("-") @@ -83,29 +73,13 @@ public class WordPatterns { return true; } - public static boolean hasWordQualities(String s) { - if (s.isBlank()) - return false; - - int start = 0; - int end = s.length(); - if (s.charAt(0) == '#') start++; - if (end > 1 && s.charAt(end-1) == '#') end--; - - for (int i = start; i < end; i++) { - char c = s.charAt(i); - if (("_@.'+-".indexOf(c) < 0) - && !(c >= 'a' && c <= 'z') - && !(c >= 'A' && c <= 'Z') - && !(c >= '0' && c <= '9') - && !(c >= '\u00C0' && c <= '\u00D6') - && !(c >= '\u00D8' && c <= '\u00f6') - && !(c >= '\u00f8' && c <= '\u00ff')) - { - return false; - } + private static boolean hasMoreThanN(String s, char c, int max) { + int idx = 0; + for (int i = 0; i <= max; i++) { + idx = s.indexOf(c, idx+1); + if (idx < 0 || idx >= s.length() - 1) + return false; } - return true; } @@ -113,10 +87,8 @@ public class WordPatterns { if (s.length() < MIN_WORD_LENGTH) { return true; } - if (!hasWordQualities(s)) { - return true; - } - if (!filter(s)) { + + if (!isNotJunkWord(s)) { return true; } diff --git a/code/libraries/ngram-bloom-filter/build.gradle b/code/libraries/ngram-bloom-filter/build.gradle deleted file mode 100644 index 1d8dabd7..00000000 --- a/code/libraries/ngram-bloom-filter/build.gradle +++ /dev/null @@ -1,42 +0,0 @@ -plugins { - id 'java' - id "io.freefair.lombok" version "5.3.3.3" - - id "de.undercouch.download" version "5.1.0" -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(17)) - } -} - -dependencies { - implementation project(':code:common:config') - implementation project(':third-party:porterstemmer') - - implementation libs.lombok - annotationProcessor libs.lombok - implementation libs.bundles.slf4j - implementation libs.notnull - - implementation libs.bundles.nlp - implementation libs.guice - implementation libs.trove - implementation libs.fastutil - - testImplementation libs.bundles.slf4j.test - testImplementation libs.bundles.junit - testImplementation libs.mockito -} - - -test { - useJUnitPlatform() -} - -task fastTests(type: Test) { - useJUnitPlatform { - excludeTags "slow" - } -} diff --git a/code/libraries/term-frequency-dict/build.gradle b/code/libraries/term-frequency-dict/build.gradle index f0d52d1f..c45b2984 100644 --- a/code/libraries/term-frequency-dict/build.gradle +++ b/code/libraries/term-frequency-dict/build.gradle @@ -21,6 +21,7 @@ dependencies { implementation project(':code:common:model') implementation project(':code:common:config') implementation project(':code:libraries:easy-lsh') + implementation project(':code:libraries:array') implementation libs.lombok annotationProcessor libs.lombok diff --git a/code/libraries/term-frequency-dict/readme.md b/code/libraries/term-frequency-dict/readme.md new file mode 100644 index 00000000..32912f0d --- /dev/null +++ b/code/libraries/term-frequency-dict/readme.md @@ -0,0 +1,12 @@ +# Term Frequency Dictionary + +This dictionary is used by various parts of the system to evaluate for example +the TF-IDF score of a keyword. + +## Central Classes + +* [TermFrequencyDict](src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java) + +## See Also + +* [tools/term-frequency-extractor](../../tools/term-frequency-extractor) constructs this file \ No newline at end of file diff --git a/code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java b/code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java index 16f40528..36a61827 100644 --- a/code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java +++ b/code/libraries/term-frequency-dict/src/main/java/nu/marginalia/term_frequency_dict/TermFrequencyDict.java @@ -1,8 +1,10 @@ package nu.marginalia.term_frequency_dict; import ca.rmen.porterstemmer.PorterStemmer; -import gnu.trove.map.hash.TLongIntHashMap; +import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; +import lombok.SneakyThrows; import nu.marginalia.LanguageModels; +import nu.marginalia.array.LongArray; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; @@ -14,39 +16,45 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Path; +/** Dictionary with term frequency information for (stemmed) words. + * + */ @Singleton public class TermFrequencyDict { - private final TLongIntHashMap wordRates = new TLongIntHashMap(1_000_000, 0.5f, 0, 0); - private final Logger logger = LoggerFactory.getLogger(getClass()); + private final Long2IntOpenHashMap wordRates; + private static final Logger logger = LoggerFactory.getLogger(TermFrequencyDict.class); private static final PorterStemmer ps = new PorterStemmer(); - private static final long DOC_COUNT_KEY = ~0L; + public static final long DOC_COUNT_KEY = ~0L; @Inject public TermFrequencyDict(@NotNull LanguageModels models) { this(models.termFrequencies); } + @SneakyThrows public TermFrequencyDict(Path file) { - try (var frequencyData = new DataInputStream(new BufferedInputStream(new FileInputStream(file.toFile())))) { - wordRates.ensureCapacity((int)(Files.size(file)/16)); - - for (;;) { - wordRates.put(frequencyData.readLong(), (int) frequencyData.readLong()); - } - } catch (EOFException eof) { - // ok - } catch (IOException e) { - logger.error("IO Exception reading " + file, e); - } + wordRates = load(file); logger.info("Read {} N-grams frequencies", wordRates.size()); } - public TermFrequencyDict(TLongIntHashMap data) { - wordRates.putAll(data); + private static Long2IntOpenHashMap load(Path file) throws IOException { + LongArray array = LongArray.mmapRead(file); + + int size = (int) Files.size(file)/16; + var ret = new Long2IntOpenHashMap(size, 0.5f); + + ret.defaultReturnValue(0); + + for (int i = 0; i < size; i++) { + ret.put(array.get(2*i), (int) array.get(2*i + 1)); + } + + return ret; } + /** Total number of documents in the corpus */ public int docCount() { int cnt = wordRates.get(DOC_COUNT_KEY); @@ -56,91 +64,20 @@ public class TermFrequencyDict { return cnt; } -// WIP refactoring, this needs a new home: -// -// public static void main(String... args) throws IOException, InterruptedException { -// if (args.length != 2) { -// System.err.println("Expected arguments: plan.yaml out-file"); -// } -// String outFile = args[1]; -// -// var plan = new CrawlPlanLoader().load(Path.of(args[0])); -// -// ThreadLocal se = ThreadLocal.withInitial(() -> new SentenceExtractor(WmsaHome.getLanguageModels())); -// LanguageFilter lf = new LanguageFilter(); -// -// TLongIntHashMap counts = new TLongIntHashMap(100_000_000, 0.7f, -1, -1); -// -// ForkJoinPool fjp = new ForkJoinPool(24); -// AtomicInteger docCount = new AtomicInteger(); -// -// for (var domain : plan.domainsIterable()) { // leaks file descriptor, is fine -// -// if (domain.doc == null) -// continue; -// -// fjp.execute(() -> { -// -// TLongHashSet words = new TLongHashSet(10_000); -// -// for (var doc : domain.doc) { -// -// if (doc.documentBody == null) -// continue; -// docCount.incrementAndGet(); -// -// Document parsed = Jsoup.parse(doc.documentBody.decode()); -// parsed.body().filter(new DomPruningFilter(0.5)); -// -// DocumentLanguageData dld = se.get().extractSentences(parsed); -// -// if (lf.dictionaryAgreement(dld) < 0.1) { -// return; -// } -// -// for (var sent : dld.sentences) { -// for (var word : sent) { -// words.add(longHash(word.stemmed().getBytes(StandardCharsets.UTF_8))); -// } -// } -// -// synchronized (counts) { -// words.forEach(w -> { -// counts.adjustOrPutValue(w, 1, 1); -// return true; -// }); -// } -// -// words.clear(); -// } -// -// System.out.println(domain.domain + "\t" + counts.size()); -// }); -// -// -// } -// -// fjp.shutdown(); -// fjp.awaitTermination(10, TimeUnit.DAYS); -// -// try (var dos = new DataOutputStream(Files.newOutputStream(Path.of(outFile)))) { -// synchronized (counts) { -// counts.put(DOC_COUNT_KEY, docCount.get()); -// -// counts.forEachEntry((hash, cnt) -> { -// try { -// dos.writeLong(hash); -// dos.writeLong(cnt); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } -// return true; -// }); -// } -// } -// -// System.out.println(docCount.get()); -// } + /** Get the term frequency for the string s */ + public long getTermFreq(String s) { + return wordRates.get(getStringHash(s)); + } + + /** Get the term frequency for the already stemmed string s */ + public long getTermFreqStemmed(String s) { + return wordRates.get(longHash(s.getBytes())); + } + + /** Get the term frequency for the already stemmed and already hashed value 'hash' */ + public long getTermFreqHash(long hash) { + return wordRates.get(hash); + } public static long getStringHash(String s) { if (s.indexOf(' ') >= 0 || s.indexOf('_') >= 0) { @@ -156,17 +93,11 @@ public class TermFrequencyDict { } } - public long getTermFreqHash(long hash) { - return wordRates.get(hash); - } - public long getTermFreq(String s) { - return wordRates.get(getStringHash(s)); - } - public long getTermFreqStemmed(String s) { - return wordRates.get(longHash(s.getBytes())); - } - - // If this ever changes, we need to re-generate the term frequency dictionary + /** The hashing function used by TermFrequencyHash + *

+ * If this function changes its behavior in any way, + * it is necessary to re-generate the dictionary. + */ public static long longHash(byte[]... bytesSets) { if (bytesSets == null || bytesSets.length == 0) return 0; diff --git a/code/process-models/converting-model/readme.md b/code/process-models/converting-model/readme.md index 59d89f2e..feaae4b3 100644 --- a/code/process-models/converting-model/readme.md +++ b/code/process-models/converting-model/readme.md @@ -1,4 +1,49 @@ # Converting Models Contains models shared by the [converting-process](../../processes/converting-process/) and -[loading-process](../../processes/loading-process/). \ No newline at end of file +[loading-process](../../processes/loading-process/). + +## Design + +The two processes communicate through a file-based protocol. The converter serializes [instructions](src/main/java/nu/marginalia/converting/instruction/Instruction.java) +to file, which are deserialized by the loader and fed into an [instructions](src/main/java/nu/marginalia/converting/instruction/Interpreter.java). + +The instructions implement a visitor pattern. + +Conceptually the pattern can be thought of a bit like remote function calls over file, +or a crude instructions-based programming language. + +This + +```java +producer.foo("cat"); +producer.bar("milk", "eggs", "bread"); +``` + +translates through this paradigm, to this: + +``` +(producer) +writeInstruction(DoFoo("Cat")) +writeInstruction(DoBar("Milk", "Eggs", "Bread")) + +(consumer) +while read instruction: + interpreter.apply(instruction) + +(Interpreter) +doFoo(animal): + ... +doBar(ingredients): + ... + +(doFoo) +DoFoo(animal): + apply(interpreter): + interpreter.foo(animal) + +(doBar) +DoBar(ingredients): + apply(interpreter): + interpreter.bar(ingredients) +``` diff --git a/code/process-models/crawling-model/readme.md b/code/process-models/crawling-model/readme.md index 3b63d74e..3f6d02a6 100644 --- a/code/process-models/crawling-model/readme.md +++ b/code/process-models/crawling-model/readme.md @@ -7,6 +7,7 @@ Contains models shared by the [crawling-process](../../processes/crawling-proces * [CrawledDocument](src/main/java/nu/marginalia/crawling/model/CrawledDocument.java) * [CrawledDomain](src/main/java/nu/marginalia/crawling/model/CrawledDomain.java) +* [CrawlingSpecification](src/main/java/nu/marginalia/crawling/model/spec/CrawlingSpecification.java) ### Marshalling * [CrawledDomainReader](src/main/java/nu/marginalia/crawling/io/CrawledDomainReader.java) diff --git a/code/libraries/language-processing/src/main/java/nu/marginalia/language/LanguageFilter.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/language/LanguageFilter.java similarity index 98% rename from code/libraries/language-processing/src/main/java/nu/marginalia/language/LanguageFilter.java rename to code/processes/converting-process/src/main/java/nu/marginalia/converting/language/LanguageFilter.java index b4ba2793..dd375fad 100644 --- a/code/libraries/language-processing/src/main/java/nu/marginalia/language/LanguageFilter.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/language/LanguageFilter.java @@ -1,4 +1,4 @@ -package nu.marginalia.language; +package nu.marginalia.converting.language; import nu.marginalia.language.encoding.UnicodeRanges; import nu.marginalia.language.model.DocumentLanguageData; diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java index a725be0a..3edd9f80 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/model/ProcessedDocument.java @@ -29,7 +29,7 @@ public class ProcessedDocument { if (details == null) return false; - return !details.metadata.hasFlag(DocumentFlags.Simple); + return true; } public OptionalDouble quality() { diff --git a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java index 4ccfdafe..fa7fd118 100644 --- a/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java +++ b/code/processes/converting-process/src/main/java/nu/marginalia/converting/processor/plugin/AbstractDocumentProcessorPlugin.java @@ -2,7 +2,7 @@ package nu.marginalia.converting.processor.plugin; import nu.marginalia.crawling.model.CrawledDocument; import nu.marginalia.crawling.model.CrawledDomain; -import nu.marginalia.language.LanguageFilter; +import nu.marginalia.converting.language.LanguageFilter; import nu.marginalia.language.model.DocumentLanguageData; import nu.marginalia.converting.model.HtmlStandard; import nu.marginalia.keyword.model.DocumentKeywordsBuilder; diff --git a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java b/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java index faf5fbb0..afedf88e 100644 --- a/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java +++ b/code/processes/converting-process/src/test/java/nu/marginalia/converting/ConvertingIntegrationTest.java @@ -4,6 +4,7 @@ package nu.marginalia.converting; import com.google.inject.Guice; import com.google.inject.Injector; import nu.marginalia.bigstring.BigString; +import nu.marginalia.converting.model.HtmlStandard; import nu.marginalia.converting.processor.DomainProcessor; import nu.marginalia.crawling.model.CrawledDocument; import nu.marginalia.crawling.model.CrawledDomain; @@ -22,8 +23,7 @@ import static org.junit.jupiter.api.Assertions.*; public class ConvertingIntegrationTest { - - DomainProcessor domainProcessor; + private DomainProcessor domainProcessor; @BeforeEach public void setUp() { @@ -60,7 +60,22 @@ public class ConvertingIntegrationTest { ret.documents.forEach(doc -> { resultsByStatusCount.merge(doc.state, 1, Integer::sum); }); - assertTrue(resultsByStatusCount.get(UrlIndexingState.OK) > 5); + + assertTrue(resultsByStatusCount.get(UrlIndexingState.OK) > 25); + + for (var doc : ret.documents) { + + if (!doc.isProcessedFully()) { + continue; + } + + var details = doc.details; + + assertTrue(details.title.length() > 4); + assertTrue(details.description.length() > 4); + assertEquals(HtmlStandard.HTML5, details.standard); + + } } private CrawledDomain readMarginaliaWorkingSet() throws IOException { diff --git a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/LanguageFilterTest.java b/code/processes/converting-process/src/test/java/nu/marginalia/converting/language/LanguageFilterTest.java similarity index 93% rename from code/processes/crawling-process/src/test/java/nu/marginalia/crawling/LanguageFilterTest.java rename to code/processes/converting-process/src/test/java/nu/marginalia/converting/language/LanguageFilterTest.java index 694810ba..f37c0cb5 100644 --- a/code/processes/crawling-process/src/test/java/nu/marginalia/crawling/LanguageFilterTest.java +++ b/code/processes/converting-process/src/test/java/nu/marginalia/converting/language/LanguageFilterTest.java @@ -1,6 +1,5 @@ -package nu.marginalia.crawling; +package nu.marginalia.converting.language; -import nu.marginalia.language.LanguageFilter; import org.jsoup.Jsoup; import org.junit.jupiter.api.Test; diff --git a/code/processes/converting-process/src/test/resources/memex-marginalia/readme.md b/code/processes/converting-process/src/test/resources/memex-marginalia/readme.md new file mode 100644 index 00000000..71e29df8 --- /dev/null +++ b/code/processes/converting-process/src/test/resources/memex-marginalia/readme.md @@ -0,0 +1,3 @@ +# Test Data + +This is a snapshot of memex.marginalia.nu from 2023-03-17. \ No newline at end of file diff --git a/code/processes/crawl-job-extractor-process/readme.md b/code/processes/crawl-job-extractor-process/readme.md deleted file mode 100644 index d8f3e98e..00000000 --- a/code/processes/crawl-job-extractor-process/readme.md +++ /dev/null @@ -1,4 +0,0 @@ -# Crawl Job Extractor - -The crawl job extractor creates a file containing a list of domains -along with known URLs. This is consumed by the [crawling-process](../crawling-process). \ No newline at end of file diff --git a/code/processes/readme.md b/code/processes/readme.md index ce86db9a..44c29a1e 100644 --- a/code/processes/readme.md +++ b/code/processes/readme.md @@ -1,22 +1,20 @@ # Processes -## 1. Crawl Job Extractor - -The [crawl-job-extractor-process](crawl-job-extractor-process/) creates a crawl job specification -based on the content in the database. - -## 2. Crawl Process +## 1. Crawl Process The [crawling-process](crawling-process/) fetches website contents and saves them as compressed JSON models described in [crawling-model](../process-models/crawling-model/). -## 3. Converting Process +The operation is specified by a crawl job specification. This is generated by [tools/crawl-job-extractor](../tools/crawl-job-extractor/) +based on the content in the database. + +## 2. Converting Process The [converting-process](converting-process/) reads crawl data from the crawling step and processes them, extracting keywords and metadata and saves them as compressed JSON models described in [converting-model](../process-models/converting-model/). -## 4. Loading Process +## 3. Loading Process The [loading-process](loading-process/) reads the processed data and creates an index journal and lexicon, and loads domains and addresses into the MariaDB-database. diff --git a/code/readme.md b/code/readme.md index a889e507..60609310 100644 --- a/code/readme.md +++ b/code/readme.md @@ -21,11 +21,15 @@ You'll find a short description in each module of what it does and how it relate Processes are batch jobs that deal with data retrieval, processing and loading. * [processes](processes/) -* * [crawl-job-extractor](processes/crawl-job-extractor-process) * * [crawling-process](processes/crawling-process) * * [converting-process](processes/converting-process) * * [loading-process](processes/loading-process) +#### Tools + +* * [crawl-job-extractor](tools/crawl-job-extractor) +* * [term-frequency-extractor](tools/term-frequency-extractor) + ### Features Features are relatively stand-alone components that serve some part of the domain. They aren't domain-independent, diff --git a/code/services-core/search-service/build.gradle b/code/services-core/search-service/build.gradle index 56f59688..60717341 100644 --- a/code/services-core/search-service/build.gradle +++ b/code/services-core/search-service/build.gradle @@ -29,7 +29,6 @@ dependencies { implementation project(':code:libraries:easy-lsh') implementation project(':code:libraries:language-processing') implementation project(':code:libraries:braille-block-punch-cards') - implementation project(':code:libraries:ngram-bloom-filter') implementation project(':code:libraries:term-frequency-dict') implementation project(':code:api:assistant-api') diff --git a/code/services-core/search-service/src/main/java/nu/marginalia/search/query/QueryFactory.java b/code/services-core/search-service/src/main/java/nu/marginalia/search/query/QueryFactory.java index d7ccde8f..50804e11 100644 --- a/code/services-core/search-service/src/main/java/nu/marginalia/search/query/QueryFactory.java +++ b/code/services-core/search-service/src/main/java/nu/marginalia/search/query/QueryFactory.java @@ -8,8 +8,8 @@ import nu.marginalia.index.client.model.query.SearchSubquery; import nu.marginalia.index.query.limit.QueryLimits; import nu.marginalia.index.query.limit.QueryStrategy; import nu.marginalia.index.query.limit.SpecificationLimit; -import nu.marginalia.language.statistics.EnglishDictionary; -import nu.marginalia.ngram_bloom_filter.NGramBloomFilter; +import nu.marginalia.language.EnglishDictionary; +import nu.marginalia.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.query_parser.QueryParser; import nu.marginalia.query_parser.QueryPermutation; diff --git a/code/services-core/search-service/src/test/java/nu/marginalia/search/query/QueryFactoryTest.java b/code/services-core/search-service/src/test/java/nu/marginalia/search/query/QueryFactoryTest.java index 1c36922f..84377154 100644 --- a/code/services-core/search-service/src/test/java/nu/marginalia/search/query/QueryFactoryTest.java +++ b/code/services-core/search-service/src/test/java/nu/marginalia/search/query/QueryFactoryTest.java @@ -2,9 +2,9 @@ package nu.marginalia.search.query; import nu.marginalia.WmsaHome; import nu.marginalia.index.query.limit.SpecificationLimitType; -import nu.marginalia.language.statistics.EnglishDictionary; +import nu.marginalia.language.EnglishDictionary; import nu.marginalia.index.client.model.query.SearchSpecification; -import nu.marginalia.ngram_bloom_filter.NGramBloomFilter; +import nu.marginalia.ngrams.NGramBloomFilter; import nu.marginalia.term_frequency_dict.TermFrequencyDict; import nu.marginalia.search.command.SearchJsParameter; import nu.marginalia.search.model.SearchProfile; diff --git a/code/processes/crawl-job-extractor-process/build.gradle b/code/tools/crawl-job-extractor/build.gradle similarity index 96% rename from code/processes/crawl-job-extractor-process/build.gradle rename to code/tools/crawl-job-extractor/build.gradle index 0aaecd62..07ba17f5 100644 --- a/code/processes/crawl-job-extractor-process/build.gradle +++ b/code/tools/crawl-job-extractor/build.gradle @@ -31,7 +31,7 @@ dependencies { implementation libs.bundles.mariadb implementation libs.guice - implementation libs.gson + implementation libs.bundles.gson implementation libs.zstd testImplementation libs.bundles.slf4j.test diff --git a/code/tools/crawl-job-extractor/readme.md b/code/tools/crawl-job-extractor/readme.md new file mode 100644 index 00000000..ea242ba2 --- /dev/null +++ b/code/tools/crawl-job-extractor/readme.md @@ -0,0 +1,6 @@ +# Crawl Job Extractor + +The crawl job extractor creates a file containing a list of domains +along with known URLs. + +This is consumed by [processes/crawling-process](../../processes/crawling-process). \ No newline at end of file diff --git a/code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobDomainExtractor.java b/code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobDomainExtractor.java similarity index 100% rename from code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobDomainExtractor.java rename to code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobDomainExtractor.java diff --git a/code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobExtractorMain.java b/code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobExtractorMain.java similarity index 100% rename from code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobExtractorMain.java rename to code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobExtractorMain.java diff --git a/code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobSpecWriter.java b/code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobSpecWriter.java similarity index 100% rename from code/processes/crawl-job-extractor-process/src/main/java/nu/marginalia/crawl/CrawlJobSpecWriter.java rename to code/tools/crawl-job-extractor/src/main/java/nu/marginalia/crawl/CrawlJobSpecWriter.java diff --git a/code/processes/crawl-job-extractor-process/src/test/java/nu/marginalia/crawl/CrawlJobSpecWriterTest.java b/code/tools/crawl-job-extractor/src/test/java/nu/marginalia/crawl/CrawlJobSpecWriterTest.java similarity index 100% rename from code/processes/crawl-job-extractor-process/src/test/java/nu/marginalia/crawl/CrawlJobSpecWriterTest.java rename to code/tools/crawl-job-extractor/src/test/java/nu/marginalia/crawl/CrawlJobSpecWriterTest.java diff --git a/code/tools/term-frequency-extractor/build.gradle b/code/tools/term-frequency-extractor/build.gradle new file mode 100644 index 00000000..966c3c4a --- /dev/null +++ b/code/tools/term-frequency-extractor/build.gradle @@ -0,0 +1,62 @@ +plugins { + id 'java' + id "io.freefair.lombok" version "5.3.3.3" + id 'application' + + id 'jvm-test-suite' +} + +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(17)) + } +} + +application { + mainClass = 'nu.marginalia.tools.TermFrequencyExtractor' + applicationName = 'term-frequency-extractor' +} + +tasks.distZip.enabled = false + +dependencies { + implementation project(':third-party:rdrpostagger') + implementation project(':third-party:porterstemmer') + implementation project(':third-party:monkey-patch-opennlp') + implementation project(':code:common:model') + implementation project(':code:common:config') + implementation project(':code:common:process') + implementation project(':code:libraries:language-processing') + implementation project(':code:libraries:term-frequency-dict') + implementation project(':code:libraries:big-string') + implementation project(':code:processes:converting-process') + implementation project(':code:process-models:crawling-model') + + implementation libs.lombok + annotationProcessor libs.lombok + implementation libs.bundles.slf4j + implementation libs.notnull + + implementation libs.guice + implementation libs.jsoup + implementation libs.trove + implementation libs.fastutil + + implementation libs.bundles.nlp + implementation libs.commons.lang3 + + testImplementation libs.bundles.slf4j.test + testImplementation libs.bundles.junit + testImplementation libs.mockito +} + + +test { + useJUnitPlatform() +} + +task fastTests(type: Test) { + useJUnitPlatform { + excludeTags "slow" + } +} diff --git a/code/tools/term-frequency-extractor/readme.md b/code/tools/term-frequency-extractor/readme.md new file mode 100644 index 00000000..dde1dff8 --- /dev/null +++ b/code/tools/term-frequency-extractor/readme.md @@ -0,0 +1,16 @@ +# Term Frequency Extractor + +Generates a term frequency dictionary file from a batch of crawl data. + +Usage: + +```shell +PATH_TO_SAMPLES=run/samples/crawl-s +export JAVA_OPTS=-Dcrawl.rootDirRewrite=/crawl:${PATH_TO_SAMPLES} + +term-frequency-extractor ${PATH_TO_SAMPLES}/plan.yaml out.dat +``` + +## See Also + +* [libraries/term-frequency-dict](../../libraries/term-frequency-dict) \ No newline at end of file diff --git a/code/tools/term-frequency-extractor/src/main/java/nu/marginalia/tools/TermFrequencyExtractor.java b/code/tools/term-frequency-extractor/src/main/java/nu/marginalia/tools/TermFrequencyExtractor.java new file mode 100644 index 00000000..ece6a507 --- /dev/null +++ b/code/tools/term-frequency-extractor/src/main/java/nu/marginalia/tools/TermFrequencyExtractor.java @@ -0,0 +1,114 @@ +package nu.marginalia.tools; + +import gnu.trove.map.hash.TLongIntHashMap; +import gnu.trove.set.hash.TLongHashSet; +import nu.marginalia.WmsaHome; +import nu.marginalia.converting.language.LanguageFilter; +import nu.marginalia.converting.processor.logic.dom.DomPruningFilter; +import nu.marginalia.language.model.DocumentLanguageData; +import nu.marginalia.language.sentence.SentenceExtractor; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import plan.CrawlPlanLoader; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static nu.marginalia.term_frequency_dict.TermFrequencyDict.DOC_COUNT_KEY; +import static nu.marginalia.term_frequency_dict.TermFrequencyDict.longHash; + +public class TermFrequencyExtractor { + + public static void main(String... args) throws IOException, InterruptedException { + if (args.length != 2) { + System.err.println("Expected arguments: plan.yaml out-file"); + return; + } + + String outFile = args[1]; + + var plan = new CrawlPlanLoader().load(Path.of(args[0])); + + ThreadLocal se = ThreadLocal.withInitial(() -> new SentenceExtractor(WmsaHome.getLanguageModels())); + LanguageFilter lf = new LanguageFilter(); + + TLongIntHashMap counts = new TLongIntHashMap(100_000_000, 0.7f, -1, -1); + + ForkJoinPool fjp = new ForkJoinPool(24); + AtomicInteger docCount = new AtomicInteger(); + + for (var domain : plan.domainsIterable()) { // leaks file descriptor, is fine + + if (domain.doc == null) + continue; + + fjp.execute(() -> { + + TLongHashSet words = new TLongHashSet(10_000); + + for (var doc : domain.doc) { + + if (doc.documentBody == null) + continue; + docCount.incrementAndGet(); + + Document parsed = Jsoup.parse(doc.documentBody.decode()); + parsed.body().filter(new DomPruningFilter(0.5)); + + DocumentLanguageData dld = se.get().extractSentences(parsed); + + if (lf.dictionaryAgreement(dld) < 0.1) { + return; + } + + for (var sent : dld.sentences) { + for (var word : sent) { + words.add(longHash(word.stemmed().getBytes(StandardCharsets.UTF_8))); + } + } + + synchronized (counts) { + words.forEach(w -> { + counts.adjustOrPutValue(w, 1, 1); + return true; + }); + } + + words.clear(); + } + + System.out.println(domain.domain + "\t" + counts.size()); + }); + + + } + + fjp.shutdown(); + fjp.awaitTermination(10, TimeUnit.DAYS); + + try (var dos = new DataOutputStream(Files.newOutputStream(Path.of(outFile)))) { + synchronized (counts) { + counts.put(DOC_COUNT_KEY, docCount.get()); + + counts.forEachEntry((hash, cnt) -> { + try { + dos.writeLong(hash); + dos.writeLong(cnt); + } catch (IOException e) { + throw new RuntimeException(e); + } + return true; + }); + } + } + + System.out.println(docCount.get()); + } + +} diff --git a/other/memex/build.gradle b/other/memex/build.gradle deleted file mode 100644 index 5e422b57..00000000 --- a/other/memex/build.gradle +++ /dev/null @@ -1,242 +0,0 @@ -plugins { - id 'java' - id "io.freefair.lombok" version "5.3.3.3" - - id "me.champeau.jmh" version "0.6.6" - id "de.undercouch.download" version "5.1.0" - - id 'jvm-test-suite' -} - -repositories { - mavenLocal() - maven { url "https://artifactory.cronapp.io/public-release/" } - maven { url "https://repo1.maven.org/maven2/" } - maven { url "https://www2.ph.ed.ac.uk/maven2/" } - maven { url "https://jitpack.io/" } - exclusiveContent { - forRepository { - maven { - url = uri("https://jitpack.io") - } - } - filter { - // Only use JitPack for the `gson-record-type-adapter-factory` library - includeModule("com.github.Marcono1234", "gson-record-type-adapter-factory") - } - } -} - -sourceSets { - e2eTest { - java { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/e2e/java') - } - resources.srcDir file('src/e2e/resources') - } - } - jmh { - java { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/jmh/java') - } - resources.srcDir file('src/jmh/resources') - } - } -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(17)) - } -} -jmhJar { - zip64 true -} -dependencies { - implementation project(':code:common:service') - implementation project(':code:common:config') - implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') - - implementation 'org.projectlombok:lombok:1.18.24' - implementation 'org.jetbrains:annotations:20.1.0' - annotationProcessor 'org.projectlombok:lombok:1.18.24' - - implementation 'com.github.jknack:handlebars:4.3.1' - implementation 'com.github.jknack:handlebars-markdown:4.2.1' - - implementation group: 'com.google.code.gson', name: 'gson', version: '2.9.0' - implementation 'io.reactivex.rxjava3:rxjava:3.1.5' - implementation "com.sparkjava:spark-core:2.9.3" - implementation 'com.opencsv:opencsv:5.6' - - implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' - implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.2' - - implementation 'org.slf4j:slf4j-api:1.7.36' - testImplementation 'org.slf4j:slf4j-jdk14:2.0.3' - - implementation 'com.google.guava:guava:31.1-jre' - implementation 'com.google.inject:guice:5.1.0' - implementation 'com.github.jnr:jnr-ffi:2.2.12' - implementation 'org.apache.httpcomponents:httpcore:4.4.15' - implementation 'org.apache.httpcomponents:httpclient:4.5.13' - - implementation group: 'com.h2database', name: 'h2', version: '2.1.210' - - implementation 'org.jsoup:jsoup:1.15.3' - - implementation 'org.mariadb.jdbc:mariadb-java-client:3.0.6' - implementation group: 'net.sf.trove4j', name: 'trove4j', version: '3.0.3' - - implementation 'com.zaxxer:HikariCP:5.0.1' - - implementation 'org.apache.opennlp:opennlp-tools:1.9.4' - implementation 'io.prometheus:simpleclient:0.16.0' - implementation 'io.prometheus:simpleclient_servlet:0.16.0' - implementation 'io.prometheus:simpleclient_httpserver:0.16.0' - implementation 'io.prometheus:simpleclient_hotspot:0.16.0' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.3' - - implementation group: 'org.yaml', name: 'snakeyaml', version: '1.30' - - implementation 'com.github.luben:zstd-jni:1.5.2-2' - implementation 'org.lz4:lz4-java:1.8.0' - - implementation 'com.github.vladimir-bukhtoyarov:bucket4j-core:7.5.0' - implementation 'de.rototor.jeuclid:jeuclid-core:3.1.14' - - implementation 'org.imgscalr:imgscalr-lib:4.2' - implementation 'org.jclarion:image4j:0.7' - - implementation 'commons-net:commons-net:3.8.0' - implementation 'org.eclipse.jgit:org.eclipse.jgit:5.12.0.202106070339-r' - implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.jsch:5.12.0.202106070339-r' - implementation 'com.jcraft:jsch:0.1.55' - - implementation group: 'it.unimi.dsi', name: 'fastutil', version: '8.5.8' - implementation 'org.roaringbitmap:RoaringBitmap:0.9.32' - - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' - testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' - testCompileOnly 'org.projectlombok:lombok:1.18.24' - testImplementation 'org.projectlombok:lombok:1.18.24' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' - - testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.5.1' - - testImplementation platform('org.testcontainers:testcontainers-bom:1.17.4') - testImplementation 'org.testcontainers:mariadb:1.17.4' - testImplementation 'org.testcontainers:junit-jupiter:1.17.4' - - e2eTestImplementation 'org.junit.jupiter:junit-jupiter-api:5.9.0' - e2eTestRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' - e2eTestImplementation 'org.projectlombok:lombok:1.18.24' - e2eTestAnnotationProcessor 'org.projectlombok:lombok:1.18.24' - e2eTestImplementation 'org.testcontainers:nginx:1.17.4' - e2eTestImplementation "org.testcontainers:junit-jupiter:1.17.2" - e2eTestImplementation 'org.testcontainers:selenium:1.17.4' - e2eTestImplementation 'org.seleniumhq.selenium:selenium-remote-driver:4.5.3' - e2eTestImplementation 'org.seleniumhq.selenium:selenium-chrome-driver:4.5.3' - - - implementation 'org.seleniumhq.selenium:selenium-chrome-driver:4.5.3' - implementation 'org.seleniumhq.selenium:selenium-java:4.5.3' - implementation 'org.sejda.imageio:webp-imageio:0.1.6' - - jmh 'org.openjdk.jmh:jmh-core:1.35' - jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.35' - - implementation 'net.agkn:hll:1.6.0' - -} - -configurations { - e2eTestImplementation.extendsFrom(testImplementation) - -} - -test { - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 - maxHeapSize = "8G" - useJUnitPlatform() -} - -task fastTests(type: Test) { - maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1 - maxHeapSize = "8G" - useJUnitPlatform { - excludeTags "slow" - } -} - -task e2eTest(type: Test) { - maxParallelForks = 1 - forkEvery = 1 - maxHeapSize = "8G" - dependsOn ':shadowJar' - dependsOn 'downloadTestData' - dependsOn 'downloadRDRModelData' - dependsOn 'downloadSentenceModelData' - dependsOn 'downloadTokenModelData' - dependsOn 'downloadTermFreqData' - dependsOn 'IP2LocationFile' - - classpath = sourceSets.e2eTest.runtimeClasspath - testClassesDirs = sourceSets.e2eTest.output.classesDirs - useJUnitPlatform { - includeTags "e2e" - } -} - -task downloadTestData(type: Download) { - src 'http://hammurabi.acc.umu.se/mirror/kiwix.org/zim/wikipedia/wikipedia_en_100_nopic_2022-05.zim' - dest file('data/test/wikipedia_en_100_nopic.zim') - overwrite false -} - -task downloadRDRModelData(type: Download) { - src (['https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.DICT', - 'https://raw.githubusercontent.com/datquocnguyen/RDRPOSTagger/master/Models/POS/English.RDR']) - dest file('data/models/') - overwrite false -} - -task downloadSentenceModelData(type: Download) { - src 'https://dlcdn.apache.org/opennlp/models/ud-models-1.0/opennlp-en-ud-ewt-sentence-1.0-1.9.3.bin' - dest file('data/models/opennlp-sentence.bin') - overwrite false -} -task downloadTokenModelData(type: Download) { - src 'https://dlcdn.apache.org/opennlp/models/ud-models-1.0/opennlp-en-ud-ewt-tokens-1.0-1.9.3.bin' - dest file('data/models/opennlp-tokens.bin') - overwrite false -} -task downloadIP2LocationFile(type: Download) { - src 'https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.CSV.ZIP' - dest file('data/models/IP2LOCATION-LITE-DB1.CSV.ZIP') - overwrite false -} -task IP2LocationFile(type: Copy) { - dependsOn 'downloadIP2LocationFile' - def zipFile = file('data/models/IP2LOCATION-LITE-DB1.CSV.ZIP') - def outputDir = file("data/models/IP2LOC") - - from zipTree(zipFile) - into outputDir -} - -task downloadTermFreqData(type: Download) { - src 'https://downloads.marginalia.nu/model/tfreq-new-algo3.bin' - dest file('data/models/tfreq-new-algo3.bin') - overwrite false -} - diff --git a/other/memex/lombok.config b/other/memex/lombok.config deleted file mode 100644 index 6aa51d71..00000000 --- a/other/memex/lombok.config +++ /dev/null @@ -1,2 +0,0 @@ -# This file is generated by the 'io.freefair.lombok' Gradle plugin -config.stopBubbling = true diff --git a/other/memex/src/main/java/nu/marginalia/memex/MemexServiceDescriptors.java b/other/memex/src/main/java/nu/marginalia/memex/MemexServiceDescriptors.java deleted file mode 100644 index 25ef5662..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/MemexServiceDescriptors.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.memex; - -import nu.marginalia.memex.auth.AuthMain; -import nu.marginalia.service.descriptor.ServiceDescriptor; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; - -import java.util.List; - -public class MemexServiceDescriptors { - public static ServiceDescriptors descriptors = new ServiceDescriptors( - List.of( - new ServiceDescriptor(ServiceId.Other_Memex, 5030), - new ServiceDescriptor (ServiceId.Other_Auth, 5003))); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthConfigurationModule.java b/other/memex/src/main/java/nu/marginalia/memex/auth/AuthConfigurationModule.java deleted file mode 100644 index e0ad33f5..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthConfigurationModule.java +++ /dev/null @@ -1,14 +0,0 @@ -package nu.marginalia.memex.auth; - -import com.google.inject.AbstractModule; -import com.google.inject.name.Names; -import nu.marginalia.service.descriptor.HostsFile; - -import java.nio.file.Path; - -public class AuthConfigurationModule extends AbstractModule { - public void configure() { - bind(Path.class).annotatedWith(Names.named("password-file")).toInstance(Path.of("/var/lib/wmsa/password.dat")); - bind(HostsFile.class).toInstance(new HostsFile()); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthMain.java b/other/memex/src/main/java/nu/marginalia/memex/auth/AuthMain.java deleted file mode 100644 index e997d777..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthMain.java +++ /dev/null @@ -1,27 +0,0 @@ -package nu.marginalia.memex.auth; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.memex.MemexServiceDescriptors; -import nu.marginalia.service.MainClass; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.service.module.ConfigurationModule; -import nu.marginalia.service.server.Initialization; - -public class AuthMain extends MainClass { - - @Inject - public AuthMain(AuthService service) { - } - - public static void main(String... args) { - MainClass.init(ServiceId.Other_Auth, args); - - Injector injector = Guice.createInjector( - new AuthConfigurationModule(), - new ConfigurationModule(MemexServiceDescriptors.descriptors, ServiceId.Other_Auth)); - injector.getInstance(AuthMain.class); - injector.getInstance(Initialization.class).setReady(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthService.java b/other/memex/src/main/java/nu/marginalia/memex/auth/AuthService.java deleted file mode 100644 index 6f9ef59d..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/auth/AuthService.java +++ /dev/null @@ -1,118 +0,0 @@ -package nu.marginalia.memex.auth; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import nu.marginalia.client.Context; -import nu.marginalia.memex.auth.model.LoginFormModel; -import nu.marginalia.memex.renderer.MustacheRenderer; -import nu.marginalia.memex.renderer.RendererFactory; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.MetricsServer; -import nu.marginalia.service.server.RateLimiter; -import nu.marginalia.service.server.Service; -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; - -import static spark.Spark.*; - -public class AuthService extends Service { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - private String password; - - private final RateLimiter rateLimiter = RateLimiter.forLogin(); - private final MustacheRenderer loginFormRenderer; - - @Inject - public AuthService(@Named("service-host") String ip, - @Named("service-port") Integer port, - @Named("password-file") Path topSecretPasswordFile, - RendererFactory rendererFactory, - Initialization initialization, - MetricsServer metricsServer) throws IOException { - - super(ip, port, initialization, metricsServer); - - password = initPassword(topSecretPasswordFile); - - loginFormRenderer = rendererFactory.renderer("auth/login"); - - Spark.path("public/api", () -> { - before((req, rsp) -> { - logger.info("{} {}", req.requestMethod(), req.pathInfo()); - }); - - post("/login", this::login); - get("/login", this::loginForm); - }); - Spark.path("api", () -> { - get("/is-logged-in", this::isLoggedIn); - }); - } - - private String initPassword(Path topSecretPasswordFile) { - if (Files.exists(topSecretPasswordFile)) { - try { - return Files.readString(topSecretPasswordFile); - } catch (IOException e) { - logger.error("Could not read password from file " + topSecretPasswordFile, e); - } - } - logger.error("Setting random password"); - return UUID.randomUUID().toString(); - } - - private Object loginForm(Request request, Response response) { - String redir = Objects.requireNonNull(request.queryParams("redirect")); - String service = Objects.requireNonNull(request.queryParams("service")); - - return loginFormRenderer.render(new LoginFormModel(service, redir)); - } - - private Object login(Request request, Response response) { - var redir = Objects.requireNonNullElse(request.queryParams("redirect"), "/"); - - if (isLoggedIn(request, response)) { - response.redirect(redir); - return ""; - } - - if (!rateLimiter.isAllowed(Context.fromRequest(request))) { - Spark.halt(429, "Too many requests"); - return null; - } - - if (Objects.equals(password, request.queryParams("password"))) { - request.session(true).attribute("logged-in", true); - response.redirect(redir); - return ""; - } - - response.status(HttpStatus.SC_FORBIDDEN); - return "

Bad password!

"; - } - - public boolean isLoggedIn(Request request, Response response) { - var session = request.session(false); - - if (null == session) { - return false; - } - - return Optional.ofNullable(session.attribute("logged-in")) - .map(Boolean.class::cast) - .orElse(false); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/auth/client/AuthClient.java b/other/memex/src/main/java/nu/marginalia/memex/auth/client/AuthClient.java deleted file mode 100644 index f2d68667..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/auth/client/AuthClient.java +++ /dev/null @@ -1,45 +0,0 @@ -package nu.marginalia.memex.auth.client; - -import com.google.gson.GsonBuilder; -import com.google.inject.Inject; -import io.reactivex.rxjava3.core.Observable; -import nu.marginalia.WmsaHome; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; -import org.apache.http.HttpStatus; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeUnit; - - -public class AuthClient extends AbstractDynamicClient { - @Inject - public AuthClient(ServiceDescriptors descriptors) { - super(descriptors.forId(ServiceId.Other_Auth), WmsaHome.getHostsFile(), new GsonBuilder()::create); - } - - public Observable isLoggedIn(Context ctx) { - return get(ctx, "/api/is-logged-in").map(Boolean::parseBoolean); - } - - public void redirectToLoginIfUnauthenticated(String domain, Request req, Response rsp) { - if (!isLoggedIn(Context.fromRequest(req)).timeout(1, TimeUnit.SECONDS).blockingFirst()) { - rsp.redirect(req.headers("X-Extern-Domain") + "/auth/login?service="+domain - +"&redirect="+ URLEncoder.encode(req.headers("X-Extern-Url"), StandardCharsets.UTF_8)); - Spark.halt(); - } - } - - - public void requireLogIn(Context ctx) { - if (!isLoggedIn(ctx).timeout(1, TimeUnit.SECONDS).blockingFirst()) { - Spark.halt(HttpStatus.SC_FORBIDDEN); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/auth/model/LoginFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/auth/model/LoginFormModel.java deleted file mode 100644 index f31876e0..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/auth/model/LoginFormModel.java +++ /dev/null @@ -1,10 +0,0 @@ -package nu.marginalia.memex.auth.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter @AllArgsConstructor -public class LoginFormModel { - public final String service; - public final String redirect; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/BadBotList.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/BadBotList.java deleted file mode 100644 index 2b879a10..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/BadBotList.java +++ /dev/null @@ -1,43 +0,0 @@ -package nu.marginalia.memex.gemini; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetAddress; -import java.util.HashSet; -import java.util.Set; - -public class BadBotList { - private final Set shitlist = new HashSet<>(); - public static final BadBotList INSTANCE = new BadBotList(); - private final Logger logger = LoggerFactory.getLogger(getClass().getSimpleName()); - - private BadBotList() {} - - public boolean isAllowed(InetAddress address) { - return !shitlist.contains(address); - } - - public boolean isQueryPermitted(InetAddress address, String query) { - if (isBadQuery(query)) { - logger.info("Banning {}", address); - shitlist.add(address); - return false; - } - return true; - } - - private boolean isBadQuery(String query) { - if (query.startsWith("GET")) { - return true; - } - if (query.startsWith("OPTIONS")) { - return true; - } - if (query.contains("mstshash")) { - return true; - } - - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiConfigurationModule.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiConfigurationModule.java deleted file mode 100644 index 2d269332..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiConfigurationModule.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.memex.gemini; - -import com.google.inject.AbstractModule; -import com.google.inject.name.Names; - -import java.nio.file.Path; - -public class GeminiConfigurationModule extends AbstractModule { - public void configure() { - bind(Path.class).annotatedWith(Names.named("gemini-server-root")).toInstance(Path.of("/var/lib/wmsa/memex-gmi")); - bind(Path.class).annotatedWith(Names.named("gemini-cert-file")).toInstance(Path.of("/var/lib/wmsa/gemini/crypto.jks")); - bind(Path.class).annotatedWith(Names.named("gemini-cert-password-file")).toInstance(Path.of("/var/lib/wmsa/gemini/password.dat")); - bind(Integer.class).annotatedWith(Names.named("gemini-server-port")).toInstance(1965); - - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiService.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiService.java deleted file mode 100644 index d5c6db9c..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiService.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.memex.gemini; - -public interface GeminiService { - String DEFAULT_FILENAME = "index.gmi"; - - void run(); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceDummy.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceDummy.java deleted file mode 100644 index 33fcffb2..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceDummy.java +++ /dev/null @@ -1,10 +0,0 @@ -package nu.marginalia.memex.gemini; - -import com.google.inject.Singleton; - -@Singleton -public class GeminiServiceDummy implements GeminiService { - @Override - public void run() { - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceImpl.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceImpl.java deleted file mode 100644 index 27b956d9..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/GeminiServiceImpl.java +++ /dev/null @@ -1,164 +0,0 @@ -package nu.marginalia.memex.gemini; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import nu.marginalia.memex.gemini.io.GeminiConnection; -import nu.marginalia.memex.gemini.io.GeminiSSLSetUp; -import nu.marginalia.memex.gemini.io.GeminiStatusCode; -import nu.marginalia.memex.gemini.io.GeminiUserException; -import nu.marginalia.memex.gemini.plugins.BareStaticPagePlugin; -import nu.marginalia.memex.gemini.plugins.Plugin; -import nu.marginalia.memex.gemini.plugins.SearchPlugin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLException; -import javax.net.ssl.SSLServerSocket; -import javax.net.ssl.SSLServerSocketFactory; -import javax.net.ssl.SSLSocket; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -@Singleton -public class GeminiServiceImpl implements GeminiService { - - public final Path serverRoot; - - private final Logger logger = LoggerFactory.getLogger(getClass().getSimpleName()); - private final Executor pool = Executors.newFixedThreadPool(32); - private final SSLServerSocket serverSocket; - - private final Plugin[] plugins; - private final BadBotList badBotList = BadBotList.INSTANCE; - - @Inject - public GeminiServiceImpl(@Named("gemini-server-root") Path serverRoot, - @Named("gemini-server-port") Integer port, - GeminiSSLSetUp sslSetUp, - BareStaticPagePlugin pagePlugin, - SearchPlugin searchPlugin) throws Exception { - this.serverRoot = serverRoot; - logger.info("Setting up crypto"); - final SSLServerSocketFactory socketFactory = sslSetUp.getServerSocketFactory(); - - serverSocket = (SSLServerSocket) socketFactory.createServerSocket(port /* 1965 */); - serverSocket.setEnabledCipherSuites(socketFactory.getSupportedCipherSuites()); - serverSocket.setEnabledProtocols(new String[] {"TLSv1.3", "TLSv1.2"}); - - logger.info("Verifying setup"); - if (!Files.exists(this.serverRoot)) { - logger.error("Could not find SERVER_ROOT {}", this.serverRoot); - System.exit(255); - } - - plugins = new Plugin[] { - pagePlugin, - searchPlugin - }; - } - - @Override - public void run() { - logger.info("Awaiting connections"); - - try { - for (;;) { - SSLSocket connection = (SSLSocket) serverSocket.accept(); - connection.setSoTimeout(10_000); - - if (!badBotList.isAllowed(connection.getInetAddress())) { - connection.close(); - } else { - pool.execute(() -> serve(connection)); - } - } - } - catch (IOException ex) { - logger.error("IO Exception in gemini server", ex); - } - } - - private void serve(SSLSocket socket) { - final GeminiConnection connection; - try { - connection = new GeminiConnection(socket); - } - catch (IOException ex) { - logger.error("Failed to create connection object", ex); - return; - } - - try { - handleRequest(connection); - } - catch (GeminiUserException ex) { - errorResponse(connection, ex.getMessage()); - } - catch (SSLException ex) { - logger.error(connection.getAddress() + " SSL error"); - connection.close(); - } - catch (Exception ex) { - errorResponse(connection, "Error"); - logger.error(connection.getAddress(), ex); - } - finally { - connection.close(); - } - } - - private void errorResponse(GeminiConnection connection, String message) { - if (connection.isConnected()) { - try { - logger.error("=> " + connection.getAddress(), message); - connection.writeStatusLine(GeminiStatusCode.ERROR_PERMANENT, message); - } - catch (IOException ex) { - logger.error("Exception while sending error", ex); - } - } - } - - private void handleRequest(GeminiConnection connection) throws Exception { - - final String address = connection.getAddress(); - logger.info("Connect: " + address); - - final Optional maybeUri = connection.readUrl(); - if (maybeUri.isEmpty()) { - logger.info("Done: {}", address); - return; - } - - final URI uri = maybeUri.get(); - logger.info("Request {}", uri); - - if (!uri.getScheme().equals("gemini")) { - throw new GeminiUserException("Unsupported protocol"); - } - - servePage(connection, uri); - logger.info("Done: {}", address); - } - - private void servePage(GeminiConnection connection, URI url) throws IOException { - String path = url.getPath(); - - for (Plugin p : plugins) { - if (p.serve(url, connection)) { - return; - } - } - - logger.error("FileNotFound {}", path); - connection.writeStatusLine(GeminiStatusCode.ERROR_TEMPORARY, "No such file"); - } - - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/client/GeminiClient.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/client/GeminiClient.java deleted file mode 100644 index 27d2a2a9..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/client/GeminiClient.java +++ /dev/null @@ -1,130 +0,0 @@ -package nu.marginalia.memex.gemini.client; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSocketFactory; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.security.cert.X509Certificate; - -/** Unstable code! */ -public class GeminiClient { - - private final SSLSocketFactory socketFactory; - - // Create a trust manager that does not validate anything - public static final TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - @Override - public void checkClientTrusted(X509Certificate[] chain, - String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, - String authType) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - } - }; - - - public static SSLSocketFactory buildSocketFactory() throws Exception { - // Install the all-trusting trust manager - final SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, trustAllCerts, new java.security.SecureRandom()); - - return sslContext.getSocketFactory(); - } - - public GeminiClient() throws Exception { - socketFactory = buildSocketFactory(); - } - - public Response get(URI uri) throws IOException { - - final int port = uri.getPort() == -1 ? 1965 : uri.getPort(); - final String host = uri.getHost(); - var requestString = String.format("%s\r\n", uri).getBytes(StandardCharsets.UTF_8); - - try (var socket = socketFactory.createSocket(host, port)) { - socket.setSoTimeout(10_000); - socket.getOutputStream().write(requestString); - - var is = socket.getInputStream(); - String statusLine = new GeminiInput(is).get(); - - int code = Integer.parseInt(statusLine.substring(0,2)); - String meta = statusLine.substring(3); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - is.transferTo(baos); - - return new Response(code, meta, baos.toByteArray()); - } - - } - - public static class Response { - public final int code; - public final String meta; - public final byte[] data; - - Response(int code, String meta, byte[] data) { - this.code = code; - this.meta = meta; - this.data = data; - } - } - - - public static class GeminiInput { - private final InputStream is; - private final byte[] buffer = new byte[1024]; - private int idx; - - final String result; - - public GeminiInput(InputStream is) throws IOException { - this.is = is; - - for (idx = 0; idx < buffer.length; idx++) { - if (hasEndOfLine()) { - result = new String(buffer, 0, idx-2, StandardCharsets.UTF_8); - return; - } - - readCharacter(); - } - - throw new RuntimeException("String too long"); - } - - public String get() { - return result; - } - - private void readCharacter() throws IOException { - int rb = is.read(); - if (-1 == rb) { - throw new RuntimeException("URL incomplete (no CR LF)"); - } - buffer[idx] = (byte) rb; - } - - public boolean hasEndOfLine() { - return idx > 2 - && buffer[idx - 1] == (byte) '\n' - && buffer[idx - 2] == (byte) '\r'; - } - - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/Gemtext.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/Gemtext.java deleted file mode 100644 index 692b65ce..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/Gemtext.java +++ /dev/null @@ -1,53 +0,0 @@ -package nu.marginalia.memex.gemini.gmi; - -import lombok.Getter; -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.parser.GemtextParser; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRenderer; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.io.IOException; -import java.io.Writer; -import java.util.Arrays; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -@Getter -public class Gemtext { - private final AbstractGemtextLine[] lines; - private final MemexNodeUrl url; - - public Gemtext(MemexNodeUrl url, String[] lines, MemexNodeHeadingId headingRoot) { - this.lines = GemtextParser.parse(lines, headingRoot); - this.url = url; - } - public Gemtext(MemexNodeUrl url, String[] lines) { - this.lines = GemtextParser.parse(lines, new MemexNodeHeadingId(0)); - this.url = url; - } - - public String render(GemtextRenderer renderer) { - return Arrays.stream(lines).map(renderer::renderLine).collect(Collectors.joining()); - } - - public void render(GemtextRenderer renderer, Writer w) throws IOException { - for (var line : lines) { - w.write(renderer.renderLine(line)); - w.write('\n'); - } - } - - public Stream stream() { - return Arrays.stream(lines); - } - - public AbstractGemtextLine get(int idx) { - return lines[idx]; - } - public int size() { - return lines.length; - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDatabase.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDatabase.java deleted file mode 100644 index 2beb1772..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDatabase.java +++ /dev/null @@ -1,71 +0,0 @@ -package nu.marginalia.memex.gemini.gmi; - -import com.google.common.collect.Sets; -import nu.marginalia.memex.gemini.gmi.line.GemtextLineVisitorAdapter; -import nu.marginalia.memex.gemini.gmi.line.GemtextLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.MemexUrl; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; - -public class GemtextDatabase extends Gemtext { - public final Map links; - - public GemtextDatabase(MemexNodeUrl url, String[] lines) { - super(url, lines); - - links = new HashMap<>(); - for (int i = 0; i < size(); i++) { - int linkIdx = i; - - get(i).visit(new GemtextLineVisitorAdapter<>() { - @Override - public Object visit(GemtextLink g) { - links.put(g.getUrl().toString(), linkIdx); - return null; - } - }); - } - } - - public Set keys() { - return links.keySet(); - } - - public Optional getLinkData(MemexUrl url) { - Integer idx = links.get(url.getUrl()); - if (idx != null) { - return - Optional.of(get(idx).mapLink(GemtextLink::getTitle).orElse("")); - } - return Optional.empty(); - } - - - public static GemtextDatabase of(MemexNodeUrl url, String[] lines) { - return new GemtextDatabase(url, lines); - } - - public static GemtextDatabase of(MemexNodeUrl url, Path file) throws IOException { - try (var s = Files.lines(file)) { - return new GemtextDatabase(url, s.toArray(String[]::new)); - } - } - - public Set difference(GemtextDatabase other) { - Set differences = new HashSet<>(); - - Sets.difference(keys(), other.keys()).stream().map(MemexNodeUrl::new).forEach(differences::add); - - Sets.intersection(keys(), other.keys()) - .stream() - .map(MemexNodeUrl::new) - .filter(url -> !Objects.equals(getLinkData(url), other.getLinkData(url))) - .forEach(differences::add); - - return differences; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDocument.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDocument.java deleted file mode 100644 index 8e347c6f..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/GemtextDocument.java +++ /dev/null @@ -1,163 +0,0 @@ -package nu.marginalia.memex.gemini.gmi; - -import lombok.Getter; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRenderer; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.gemini.gmi.line.*; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeTaskId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.MemexTaskState; -import org.apache.commons.lang3.tuple.Pair; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Getter -public class GemtextDocument extends Gemtext { - private final Map headings; - private final Map> headingsByName; - private final Set pragmas; - private final List tasks; - - private final String title; - private final String date; - private final List links; - private final int hashCode; - - private static final Pattern datePattern = Pattern.compile(".*(\\d{4}-\\d{2}-\\d{2}).*"); - private static final GemtextRenderer rawRenderer = new GemtextRendererFactory().gemtextRendererAsIs(); - - public GemtextDocument(MemexNodeUrl url, String[] lines, MemexNodeHeadingId headingRoot) { - super(url, lines, headingRoot); - - this.hashCode = Arrays.hashCode(lines); - - GemtextDataExtractor extractor = new GemtextDataExtractor(); - - Arrays.stream(this.getLines()).forEach(extractor::take); - - this.headings = extractor.getHeadings(); - this.links = extractor.getLinks(); - this.title = Objects.requireNonNullElse(extractor.getTitle(), url.getUrl()); - this.pragmas = extractor.getPragmas(); - this.headingsByName = extractor.getHeadingsByName(); - this.tasks = extractor.getTasks(); - this.date = extractor.getDate(); - } - - public String getHeadingForElement(AbstractGemtextLine line) { - return headings.getOrDefault(line.getHeading(), ""); - } - - public List getSection(MemexNodeHeadingId headingId) { - return stream() - .filter(line -> line.getHeading().isChildOf(headingId)) - .collect(Collectors.toList()); - } - - public String getSectionGemtext(MemexNodeHeadingId headingId) { - if (headingId.equals(new MemexNodeHeadingId(0))) { - return stream() - .map(rawRenderer::renderLine) - .collect(Collectors.joining("\n")); - } - - return stream() - .filter(line -> line.getHeading().isChildOf(headingId)) - .map(rawRenderer::renderLine) - .collect(Collectors.joining("\n")); - } - - public Map> getOpenTopTasks() { - return tasks.stream() - .filter(task -> MemexTaskState.TODO.equals(task.getState()) - || MemexTaskState.URGENT.equals(task.getState())) - .filter(task -> task.getId().level() == 1) - .collect(Collectors.toMap(GemtextTask::getId, task -> Pair.of(task.getTask(), task.getState()))); - } - - public static GemtextDocument of(MemexNodeUrl url, String... lines) { - return new GemtextDocument(url, lines, new MemexNodeHeadingId(0)); - } - - public static GemtextDocument of(MemexNodeUrl url, Path file) throws IOException { - try (var s = Files.lines(file)) { - return new GemtextDocument(url, s.toArray(String[]::new), new MemexNodeHeadingId(0)); - } - } - - public boolean isIndex() { - return getUrl().getFilename().equals("index.gmi"); - } - - @Override - public int hashCode() { - return hashCode; - } - - public Optional getHeading(MemexNodeHeadingId heading) { - return Optional.ofNullable(headings.get(heading)); - } - - public Optional getHeadingByName(MemexNodeHeadingId parent, String name) { - var headings = headingsByName.get(name); - if (null == headings) { - return Optional.empty(); - } - return headings.stream().filter(heading -> heading.isChildOf(parent)).findAny(); - } - - @Getter - private static class GemtextDataExtractor extends GemtextLineVisitorAdapter { - - private String title; - private String date; - private final Map headings = new TreeMap<>((a, b) -> Arrays.compare(a.getIds(), b.getIds())); - private final Map> headingsByName = new HashMap<>(); - private final Set pragmas = new HashSet<>(); - private final List links = new ArrayList<>(); - private final List tasks = new ArrayList<>(); - - @Override - public Object visit(GemtextHeading g) { - headings.put(g.getLevel(), g.getName()); - headingsByName.computeIfAbsent(g.getName(), t -> new ArrayList<>()).add(g.getLevel()); - - if (title == null) { - title = g.getName(); - var dateMatcher = datePattern.matcher(title); - if (dateMatcher.matches()) { - date = dateMatcher.group(1); - } - } - - return null; - } - - @Override - public Object visit(GemtextLink g) { - links.add(g); - - return null; - } - - @Override - public Object visit(GemtextTask g) { - tasks.add(g); - - return null; - } - - @Override - public Object visit(GemtextPragma g) { - pragmas.add(g.getLine()); - - return null; - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/AbstractGemtextLine.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/AbstractGemtextLine.java deleted file mode 100644 index 21e0bf4d..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/AbstractGemtextLine.java +++ /dev/null @@ -1,18 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.Optional; -import java.util.function.Function; - -public abstract class AbstractGemtextLine { - public Optional mapLink(Function mapper) { - return Optional.empty(); - } - public Optional mapHeading(Function mapper) { return Optional.empty(); } - public Optional mapTask(Function mapper) { return Optional.empty(); } - public abstract T visit(GemtextLineVisitor visitor); - - public abstract boolean breaksTask(); - public abstract MemexNodeHeadingId getHeading(); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextAside.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextAside.java deleted file mode 100644 index 9c0b5632..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextAside.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -@AllArgsConstructor @Getter @ToString -public class GemtextAside extends AbstractGemtextLine { - private final String line; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextHeading.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextHeading.java deleted file mode 100644 index d969bc95..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextHeading.java +++ /dev/null @@ -1,32 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.Optional; -import java.util.function.Function; - -@AllArgsConstructor -@Getter -@ToString -public class GemtextHeading extends AbstractGemtextLine { - private final MemexNodeHeadingId level; - private final String name; - private final MemexNodeHeadingId heading; - - public Optional mapHeading(Function mapper) { - return Optional.of(mapper.apply(this)); - } - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return true; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitor.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitor.java deleted file mode 100644 index ef3cb97a..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitor.java +++ /dev/null @@ -1,18 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -public interface GemtextLineVisitor { - default T take(AbstractGemtextLine line) { - return line.visit(this); - } - - T visit(GemtextHeading g); - T visit(GemtextLink g); - T visit(GemtextList g); - T visit(GemtextPreformat g); - T visit(GemtextQuote g); - T visit(GemtextText g); - T visit(GemtextTextLiteral g); - T visit(GemtextAside g); - T visit(GemtextTask g); - T visit(GemtextPragma g); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitorAdapter.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitorAdapter.java deleted file mode 100644 index 8e00aae3..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLineVisitorAdapter.java +++ /dev/null @@ -1,53 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -public class GemtextLineVisitorAdapter implements GemtextLineVisitor { - @Override - public T visit(GemtextHeading g) { - return null; - } - - @Override - public T visit(GemtextLink g) { - return null; - } - - @Override - public T visit(GemtextList g) { - return null; - } - - @Override - public T visit(GemtextPreformat g) { - return null; - } - - @Override - public T visit(GemtextQuote g) { - return null; - } - - @Override - public T visit(GemtextText g) { - return null; - } - - @Override - public T visit(GemtextTextLiteral g) { - return null; - } - - @Override - public T visit(GemtextAside g) { - return null; - } - - @Override - public T visit(GemtextTask g) { - return null; - } - - @Override - public T visit(GemtextPragma g) { - return null; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLink.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLink.java deleted file mode 100644 index f4359946..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextLink.java +++ /dev/null @@ -1,33 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexUrl; - -import javax.annotation.Nullable; -import java.util.Optional; -import java.util.function.Function; - -@AllArgsConstructor @Getter @ToString -public class GemtextLink extends AbstractGemtextLine { - private final MemexUrl url; - - @Nullable - private final String title; - private final MemexNodeHeadingId heading; - - public Optional mapLink(Function mapper) { - return Optional.ofNullable(mapper.apply(this)); - } - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextList.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextList.java deleted file mode 100644 index 903b4f63..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextList.java +++ /dev/null @@ -1,23 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.List; - -@AllArgsConstructor @Getter @ToString -public class GemtextList extends AbstractGemtextLine { - private final List items; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return true; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPragma.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPragma.java deleted file mode 100644 index 1baf6658..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPragma.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -@AllArgsConstructor @Getter @ToString -public class GemtextPragma extends AbstractGemtextLine { - private final String line; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPreformat.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPreformat.java deleted file mode 100644 index a83e70bc..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextPreformat.java +++ /dev/null @@ -1,23 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.List; - -@AllArgsConstructor @Getter @ToString -public class GemtextPreformat extends AbstractGemtextLine { - private final List items; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return true; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextQuote.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextQuote.java deleted file mode 100644 index 5eb318a2..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextQuote.java +++ /dev/null @@ -1,23 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.List; - -@AllArgsConstructor @Getter @ToString -public class GemtextQuote extends AbstractGemtextLine { - private final List items; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return true; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTask.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTask.java deleted file mode 100644 index 4d371785..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTask.java +++ /dev/null @@ -1,42 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeTaskId; -import nu.marginalia.memex.memex.model.MemexTaskState; -import nu.marginalia.memex.memex.model.MemexTaskTags; - -import java.util.Optional; -import java.util.function.Function; - -@AllArgsConstructor @Getter @ToString -public class GemtextTask extends AbstractGemtextLine { - private final MemexNodeTaskId id; - private final String task; - private final MemexNodeHeadingId heading; - private final MemexTaskTags tags; - - public MemexTaskState getState() { - return MemexTaskState.of(tags); - } - - public int getLevel() { - return id.level(); - } - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - @Override - public boolean breaksTask() { - return true; - } - - @Override - public Optional mapTask(Function mapper) { - return Optional.of(mapper.apply(this)); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextText.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextText.java deleted file mode 100644 index 80a84e7e..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextText.java +++ /dev/null @@ -1,21 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -@AllArgsConstructor @Getter @ToString -public class GemtextText extends AbstractGemtextLine { - private final String line; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return !line.isBlank(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTextLiteral.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTextLiteral.java deleted file mode 100644 index d2a6698e..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/line/GemtextTextLiteral.java +++ /dev/null @@ -1,23 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.line; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.List; - -@AllArgsConstructor @Getter @ToString -public class GemtextTextLiteral extends AbstractGemtextLine { - private final List items; - private final MemexNodeHeadingId heading; - - @Override - public T visit(GemtextLineVisitor visitor) { - return visitor.visit(this); - } - - public boolean breaksTask() { - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextAsideParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextAsideParser.java deleted file mode 100644 index 3038111d..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextAsideParser.java +++ /dev/null @@ -1,20 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.GemtextAside; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.regex.Pattern; - -public class GemtextAsideParser { - private static final Pattern listItemPattern = Pattern.compile("^\\((.*)\\)$"); - - public static GemtextAside parse(String s, MemexNodeHeadingId heading) { - var matcher = listItemPattern.matcher(s); - - if (!matcher.matches()) { - return null; - } - - return new GemtextAside(matcher.group(1), heading); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextHeadingParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextHeadingParser.java deleted file mode 100644 index 9a0c5329..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextHeadingParser.java +++ /dev/null @@ -1,26 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextHeading; -import nu.marginalia.memex.gemini.gmi.line.GemtextText; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.regex.Pattern; - -public class GemtextHeadingParser { - private static final Pattern headingPattern = Pattern.compile("^(#+)\\s*([^#].*|$)$"); - - public static AbstractGemtextLine parse(String s, MemexNodeHeadingId heading) { - var matcher = headingPattern.matcher(s); - - if (!matcher.matches()) { - return new GemtextText(s, heading); - } - - int level = matcher.group(1).length() - 1; - var newHeading = heading.next(level); - - return new GemtextHeading(newHeading, matcher.group(2), newHeading); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextLinkParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextLinkParser.java deleted file mode 100644 index 04b5353a..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextLinkParser.java +++ /dev/null @@ -1,42 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextLink; -import nu.marginalia.memex.gemini.gmi.line.GemtextText; -import nu.marginalia.memex.memex.model.MemexExternalUrl; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.MemexUrl; - -import javax.annotation.Nullable; -import java.util.regex.Pattern; - -public class GemtextLinkParser { - private static final Pattern linkPattern = Pattern.compile("^=>\\s?([^\\s]+)\\s*(.+)?$"); - - @Nullable - public static AbstractGemtextLine parse(String s, MemexNodeHeadingId heading) { - var matcher = linkPattern.matcher(s); - - if (!matcher.matches()) { - return new GemtextText(s, heading); - } - if (matcher.groupCount() == 2) { - return new GemtextLink(toMemexUrl(matcher.group(1)), matcher.group(2), heading); - } - else { - return new GemtextLink(toMemexUrl(matcher.group(1)), null, heading); - } - } - - private static MemexUrl toMemexUrl(String url) { - if (url.startsWith("/")) { - return new MemexNodeUrl(url); - } - else { - return new MemexExternalUrl(url); - } - } - - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextListParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextListParser.java deleted file mode 100644 index 8a93de29..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextListParser.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import java.util.regex.Pattern; - -public class GemtextListParser { - private static final Pattern listItemPattern = Pattern.compile("^\\*\\s?(.+)$"); - - public static String parse(String s) { - var matcher = listItemPattern.matcher(s); - - if (!matcher.matches()) { - return null; - } - - return matcher.group(1); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextParser.java deleted file mode 100644 index d1b63a69..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextParser.java +++ /dev/null @@ -1,135 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.*; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeTaskId; - -import java.util.*; - -public class GemtextParser { - - private static final String PREFORMAT_MARKER = "```"; - private static final String LITERAL_MARKER = " "; - private static final String LINK_MARKER = "=>"; - private static final String HEADING_MARKER = "#"; - private static final String LIST_MARKER = "*"; - private static final String QUOTE_MARKER = ">"; - private static final String ASIDE_MARKER = "("; - private static final String TASK_MARKER = "-"; - private static final String PRAGMA_MARKER = "%%%"; - - public static AbstractGemtextLine[] parse(String[] lines, MemexNodeHeadingId headingRoot) { - List items = new ArrayList<>(); - MemexNodeHeadingId heading = headingRoot; - MemexNodeTaskId task = new MemexNodeTaskId(0); - - Set pragmas = new HashSet<>(); - - for (int i = 0; i < lines.length; i++) { - String line = lines[i]; - - if (line.startsWith(PREFORMAT_MARKER)) { - i = getBlockQuote(items, lines, heading, i); - } - else if (line.startsWith(PRAGMA_MARKER)) { - var pragma = GemtextPragmaParser.parse(line, heading); - - if (pragma instanceof GemtextPragma) { - GemtextPragma gtp = (GemtextPragma) pragma; - pragmas.add(gtp.getLine()); - } - - items.add(pragma); - - } - else if (line.startsWith(LINK_MARKER)) { - var link = GemtextLinkParser.parse(line, heading); - items.add(link); - } - else if (line.startsWith(HEADING_MARKER)) { - var tag = GemtextHeadingParser.parse(line, heading); - - heading = tag.mapHeading(GemtextHeading::getHeading).orElse(heading); - - items.add(tag); - } - else if (line.startsWith(LIST_MARKER)) { - i = getList(items, lines, heading, i); - } - else if (line.startsWith(LITERAL_MARKER)) { - i = getLitteral(items, lines, heading, i); - } - else if (pragmas.contains("TASKS") - && line.startsWith(TASK_MARKER)) - { - var tag = GemtextTaskParser.parse(line, heading, task); - - task = tag.mapTask(GemtextTask::getId).orElse(task); - - items.add(tag); - } - else if (line.startsWith(QUOTE_MARKER)) { - i = getQuote(items, lines, heading, i); - } - else if (line.startsWith(ASIDE_MARKER)) { - var aside = GemtextAsideParser.parse(line, heading); - items.add(Objects.requireNonNullElse(aside, new GemtextText(line, heading))); - } - else { - items.add(new GemtextText(line, heading)); - } - } - return items.toArray(AbstractGemtextLine[]::new); - } - - private static int getBlockQuote(List items, String[] lines, MemexNodeHeadingId heading, int i) { - int j = i+1; - List quotedLines = new ArrayList<>(); - for (;j < lines.length; j++) { - if (lines[j].startsWith(PREFORMAT_MARKER)) { - break; - } - quotedLines.add(lines[j]); - } - items.add(new GemtextPreformat(quotedLines, heading)); - return j; - } - - private static int getList(List items, String[] lines, MemexNodeHeadingId heading, int i) { - int j = i; - List listLines = new ArrayList<>(); - for (;j < lines.length; j++) { - if (!lines[j].startsWith(LIST_MARKER)) { - break; - } - listLines.add(GemtextListParser.parse(lines[j])); - } - items.add(new GemtextList(listLines, heading)); - return j-1; - } - private static int getLitteral(List items, String[] lines, MemexNodeHeadingId heading, int i) { - int j = i; - List listLines = new ArrayList<>(); - for (;j < lines.length; j++) { - if (!lines[j].startsWith(LITERAL_MARKER)) { - break; - } - listLines.add(lines[j]); - } - items.add(new GemtextTextLiteral(listLines, heading)); - return j-1; - } - - private static int getQuote(List items, String[] lines, MemexNodeHeadingId heading, int i) { - int j = i; - List listLines = new ArrayList<>(); - for (;j < lines.length; j++) { - if (!lines[j].startsWith(QUOTE_MARKER)) { - break; - } - listLines.add(GemtextQuoteParser.parse(lines[j])); - } - items.add(new GemtextQuote(listLines, heading)); - return j-1; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextPragmaParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextPragmaParser.java deleted file mode 100644 index f96581cc..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextPragmaParser.java +++ /dev/null @@ -1,26 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextPragma; -import nu.marginalia.memex.gemini.gmi.line.GemtextText; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; - -import java.util.regex.Pattern; - -public class GemtextPragmaParser { - private static final Pattern pragmaPattern = Pattern.compile("^%%%\\s*(.*|$)$"); - - public static AbstractGemtextLine parse(String s, MemexNodeHeadingId heading) { - var matcher = pragmaPattern.matcher(s); - - if (!matcher.matches()) { - return new GemtextText(s, heading); - } - - String task = matcher.group(1); - - return new GemtextPragma(task, heading); - } - - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextQuoteParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextQuoteParser.java deleted file mode 100644 index b4468ea3..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextQuoteParser.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import java.util.regex.Pattern; - -public class GemtextQuoteParser { - private static final Pattern listItemPattern = Pattern.compile("^>(.+)$"); - - public static String parse(String s) { - var matcher = listItemPattern.matcher(s); - - if (!matcher.matches()) { - return null; - } - - return matcher.group(1); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextTaskParser.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextTaskParser.java deleted file mode 100644 index 08b1b803..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/parser/GemtextTaskParser.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextTask; -import nu.marginalia.memex.gemini.gmi.line.GemtextText; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeTaskId; -import nu.marginalia.memex.memex.model.MemexTaskTags; - -import java.util.regex.Pattern; - -public class GemtextTaskParser { - private static final Pattern taskPattern = Pattern.compile("^(-+)\\s*([^-].*|$)$"); - - public static AbstractGemtextLine parse(String s, MemexNodeHeadingId heading, - MemexNodeTaskId taskId) { - var matcher = taskPattern.matcher(s); - - if (!matcher.matches()) { - return new GemtextText(s, heading); - } - - int level = matcher.group(1).length() - 1; - - String task = matcher.group(2); - - return new GemtextTask(taskId.next(level), task, heading, new MemexTaskTags(task)); - } - - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRenderer.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRenderer.java deleted file mode 100644 index ccfa1fd7..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRenderer.java +++ /dev/null @@ -1,91 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.renderer; - -import nu.marginalia.memex.gemini.gmi.line.*; - -import java.util.function.Function; - -public class GemtextRenderer implements GemtextLineVisitor { - - private final Function headingConverter; - private final Function linkConverter; - private final Function listConverter; - private final Function preformatConverter; - private final Function quoteConverter; - private final Function textConverter; - private final Function asideConverter; - private final Function taskConverter; - private final Function literalConverter; - private final Function pragmaConverter; - - public GemtextRenderer(Function headingConverter, - Function linkConverter, - Function listConverter, - Function preformatConverter, - Function quoteConverter, - Function textConverter, - Function asideConverter, - Function taskConverter, - Function literalConverter, - Function pragmaConverter - ) { - this.headingConverter = headingConverter; - this.linkConverter = linkConverter; - this.listConverter = listConverter; - this.preformatConverter = preformatConverter; - this.quoteConverter = quoteConverter; - this.textConverter = textConverter; - this.asideConverter = asideConverter; - this.taskConverter = taskConverter; - this.literalConverter = literalConverter; - this.pragmaConverter = pragmaConverter; - } - - - public String renderLine(AbstractGemtextLine line) { - return line.visit(this); - } - - @Override - public String visit(GemtextHeading g) { - return headingConverter.apply(g); - } - - @Override - public String visit(GemtextLink g) { - return linkConverter.apply(g); - } - - @Override - public String visit(GemtextList g) { - return listConverter.apply(g); - } - - @Override - public String visit(GemtextPreformat g) { - return preformatConverter.apply(g); - } - - @Override - public String visit(GemtextQuote g) { - return quoteConverter.apply(g); - } - - @Override - public String visit(GemtextText g) { - return textConverter.apply(g); - } - - @Override - public String visit(GemtextTextLiteral g) { - return literalConverter.apply(g); - } - - @Override - public String visit(GemtextAside g) { return asideConverter.apply(g); } - - @Override - public String visit(GemtextTask g) { return taskConverter.apply(g); } - - @Override - public String visit(GemtextPragma g) { return pragmaConverter.apply(g); } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRendererFactory.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRendererFactory.java deleted file mode 100644 index 0e75d047..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/gmi/renderer/GemtextRendererFactory.java +++ /dev/null @@ -1,227 +0,0 @@ -package nu.marginalia.memex.gemini.gmi.renderer; - -import nu.marginalia.memex.gemini.gmi.line.*; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.MemexUrl; -import org.apache.logging.log4j.util.Strings; - -import java.util.Objects; -import java.util.stream.Collectors; - -public class GemtextRendererFactory { - - public final String urlBase; - public final String docUrl; - - public GemtextRendererFactory(String urlBase, String docUrl) { - this.urlBase = Objects.requireNonNull(urlBase, "urlBase must not be null"); - this.docUrl = Objects.requireNonNull(docUrl, "docUrl must not be null"); - } - - public GemtextRendererFactory(String urlBase) { - this.urlBase = Objects.requireNonNull(urlBase, "urlBase must not be null"); - this.docUrl = null; - } - - public GemtextRendererFactory() { - this.urlBase = null; - this.docUrl = null; - } - - public GemtextRenderer htmlRendererEditable() { - return new GemtextRenderer(this::htmlHeadingEditable, - this::htmlLink, this::htmlList, - this::htmlPre, this::htmlQuote, - this::htmlText, this::htmlAside, - this::htmlTask, this::htmlLiteral, - this::htmlPragma); - } - - public GemtextRenderer htmlRendererReadOnly() { - return new GemtextRenderer(this::htmlHeadingReadOnly, - this::htmlLink, this::htmlList, - this::htmlPre, this::htmlQuote, - this::htmlText, this::htmlAside, - this::htmlTask, this::htmlLiteral, - this::htmlPragma); - } - - - public GemtextRenderer gemtextRendererAsIs() { - return new GemtextRenderer(this::rawHeading, - this::rawLink, this::rawList, - this::rawPre, this::rawQuote, - this::rawText, this::rawAside, - this::rawTask, this::rawLiteral, - this::rawPragma); - } - - - public GemtextRenderer gemtextRendererPublic() { - return new GemtextRenderer(this::rawHeading, - this::rawLink, this::rawList, - this::rawPre, this::rawQuote, - this::rawText, this::rawAside, - this::rawTask, this::rawLiteral, - this::rawSupressPragma); - } - - - private String htmlPragma(GemtextPragma gemtextPragma) { - return "\n"; - } - - public String htmlHeadingEditable(GemtextHeading g) { - if (docUrl == null) { - throw new UnsupportedOperationException("Wrong constructor used, need urlBase and docUrl"); - } -// String editLink = String.format("\nEdit\n", urlBase + docUrl, g.getLevel()); - - return htmlHeadingReadOnly(g); - } - - public String htmlHeadingReadOnly(GemtextHeading g) { - if (g.getLevel().getLevel() == 1) - return String.format("

%s

\n", g.getLevel(), sanitizeText(g.getName())); - if (g.getLevel().getLevel() == 2) - return String.format("

%s

\n", g.getLevel(), sanitizeText(g.getName())); - if (g.getLevel().getLevel() == 3) - return String.format("

%s

\n", g.getLevel(), sanitizeText(g.getName())); - - return String.format("

%s

\n", g.getLevel(), sanitizeText(g.getName())); - } - - public String htmlLink(GemtextLink g) { - if (urlBase == null) { - throw new UnsupportedOperationException("Wrong constructor used, need urlBase"); - } - final String linkClass = getLinkClass(g.getUrl()); - final String linkUrl = getLinkUrl(g.getUrl()).replaceFirst("^gemini://", "https://proxy.vulpes.one/gemini/"); - if (g.getTitle() != null) { - return String.format("
%s
%s
\n", - linkClass, linkUrl, g.getUrl(), sanitizeText(g.getTitle())); - } - else { - return String.format("%s
\n", - linkClass, linkUrl, g.getUrl()); - } - } - private String getLinkUrl(MemexUrl url) { - if (url instanceof MemexNodeUrl || url.getUrl().startsWith("/")) { - return urlBase + url; - } - return url.toString(); - } - - private String getLinkClass(MemexUrl url) { - if (url instanceof MemexNodeUrl) { - return "internal"; - } - return "external"; - } - public String htmlList(GemtextList g) { - return g.getItems() - .stream() - .map(s -> "
  • " + sanitizeText(s) + "
    • ") - .collect( - Collectors.joining("\n", "
      \n", "
    \n")); - } - - public String htmlPre(GemtextPreformat g) { - return g.getItems().stream() - .map(this::sanitizeText) - .collect( - Collectors.joining("\n", "
    \n", "
    \n")); - } - - public String htmlLiteral(GemtextTextLiteral g) { - return g.getItems().stream() - .map(this::sanitizeText) - .collect( - Collectors.joining("\n", "
    \n", "
    \n")); - } - public String htmlQuote(GemtextQuote g) { - return g.getItems().stream() - .map(this::sanitizeText) - .collect( - Collectors.joining("
    \n", "
    \n", "
    \n")); - - } - public String htmlText(GemtextText g) { - return sanitizeText(g.getLine()) + "
    \n"; - } - public String htmlAside(GemtextAside g) { - return "\n"; - } - - public String sanitizeText(String s) { - return s.replaceAll("<", "<").replaceAll(">", ">"); - } - - public String htmlTask(GemtextTask g) { - return String.format("
    %s %s
    \n", - g.getId(), - g.getState().style, - g.getId(), - "-".repeat(g.getLevel()), - g.getTask()); - } - - public String rawHeading(GemtextHeading g) { - if (g.getLevel().getLevel() == 1) - return "# " + g.getName(); - if (g.getLevel().getLevel() == 2) - return "## " + g.getName(); - if (g.getLevel().getLevel() == 3) - return "### " + g.getName(); - - return "### " + g.getName(); - } - - public String rawLink(GemtextLink g) { - if (g.getTitle() != null && !g.getTitle().isBlank()) { - return "=> " + g.getUrl().getUrl() + "\t" + g.getTitle(); - } - return "=> " + g.getUrl().getUrl(); - } - - public String rawList(GemtextList g) { - return g.getItems() - .stream() - .map(s -> "* " + s) - .collect(Collectors.joining("\n")); - } - - public String rawPre(GemtextPreformat g) { - return g.getItems().stream() - .collect(Collectors.joining("\n", "```\n", "\n```")); - } - - public String rawQuote(GemtextQuote g) { - return g.getItems().stream() - .map(s -> "> " + s) - .collect(Collectors.joining()); - - } - - public String rawText(GemtextText g) { - return g.getLine(); - } - - public String rawLiteral(GemtextTextLiteral g) { - return Strings.join(g.getItems(), '\n'); - } - - public String rawAside(GemtextAside g) { - return "(" + g.getLine() + ")"; - } - public String rawTask(GemtextTask g) { - return "-".repeat(Math.max(0, g.getLevel())) + " " + g.getTask(); - } - private String rawPragma(GemtextPragma gemtextPragma) { - return "%%% " + gemtextPragma.getLine(); - } - private String rawSupressPragma(GemtextPragma gemtextPragma) { - return ""; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiConnection.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiConnection.java deleted file mode 100644 index 3278ec2f..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiConnection.java +++ /dev/null @@ -1,185 +0,0 @@ -package nu.marginalia.memex.gemini.io; - -import nu.marginalia.memex.gemini.BadBotList; -import nu.marginalia.memex.gemini.plugins.FileType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLSocket; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.stream.Stream; - -public class GeminiConnection { - private final SSLSocket connection; - - private final Logger logger = LoggerFactory.getLogger("Server"); - private final OutputStream os; - private final InputStream is; - private static final BadBotList badBotList = BadBotList.INSTANCE; - - public GeminiConnection(SSLSocket connection) throws IOException { - this.connection = connection; - - this.os = connection.getOutputStream(); - this.is = connection.getInputStream(); - - } - - public String getAddress() { - return connection.getInetAddress().getHostAddress(); - } - - public Optional readUrl() throws Exception { - - var str = new GeminiInput().get(); - if (!badBotList.isQueryPermitted(connection.getInetAddress(), str)) { - return Optional.empty(); - } - if (!str.isBlank()) { - return Optional.of(new URI(str)); - } - throw new GeminiUserException("Bad URI"); - } - - public void redirect(String address) throws IOException { - writeStatusLine(GeminiStatusCode.REDIRECT, address); - } - public void redirectPermanent(String address) throws IOException { - writeStatusLine(GeminiStatusCode.REDIRECT_PERMANENT, address); - } - public GeminiConnection writeStatusLine(int code, String meta) throws IOException { - write(String.format("%2d %s", code, meta)); - return this; - } - - public GeminiConnection writeBytes(byte[] data) throws IOException { - write(data); - return this; - } - - public GeminiConnection printf(String pattern, Object...args) throws IOException { - write(String.format(pattern, args)); - return this; - } - - public GeminiConnection writeLines(String... lines) throws IOException { - for (String s : lines) { - write(s); - } - return this; - } - public GeminiConnection writeLinesFromFile(Path file) throws IOException { - try (Stream lines = Files.lines(file)) { - lines.forEach(line -> { - try { - write(line); - } catch (IOException e) { - logger.error("IO Error", e); - } - }); - } - return this; - } - - public GeminiConnection acceptLines(Stream lines) { - lines.forEach(line -> { - try { - write(line); - } catch (IOException e) { - logger.error("IO exception", e); - } - }); - return this; - } - - private void write(String s) throws IOException { - os.write(s.getBytes(StandardCharsets.UTF_8)); - os.write(new byte[] { '\r', '\n'}); - } - - private void write(byte[] bs) throws IOException { - os.write(bs); - } - // This is a weird pattern but it makes the listing code very much cleaner - - public void error(String message) { - logger.error("{}", message); - - throw new GeminiUserException(message); - } - - public void close() { - try { - connection.shutdownOutput(); - connection.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - public boolean isConnected() { - return connection.isConnected(); - } - - public void respondWithFile(Path serverPath, FileType fileType) throws IOException { - if (fileType.binary) { - writeStatusLine(GeminiStatusCode.SUCCESS, fileType.mime) - .writeBytes(Files.readAllBytes(serverPath)); - } - else { - writeStatusLine(GeminiStatusCode.SUCCESS, fileType.mime) - .writeLinesFromFile(serverPath); - } - } - - public class GeminiInput { - private final byte[] buffer = new byte[1024]; - private int idx = 0; - - final String result; - - public GeminiInput() throws IOException { - - for (idx = 0; idx < buffer.length; idx++) { - if (hasEndOfLine()) { - result = new String(buffer, 0, idx-2, StandardCharsets.UTF_8); - return; - } - - readCharacter(); - } - - error("String too long"); - - // unreachable - result = ""; - } - - public String get() { - return result; - } - - private void readCharacter() throws IOException { - int rb = is.read(); - if (-1 == rb) { - error("URL incomplete (no CR LF)"); - } - buffer[idx] = (byte) rb; - } - - public boolean hasEndOfLine() { - return idx > 2 - && buffer[idx - 1] == (byte) '\n' - && buffer[idx - 2] == (byte) '\r'; - } - - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiSSLSetUp.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiSSLSetUp.java deleted file mode 100644 index cd8afa3e..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiSSLSetUp.java +++ /dev/null @@ -1,49 +0,0 @@ -package nu.marginalia.memex.gemini.io; - -import com.google.inject.Inject; -import com.google.inject.name.Named; - -import javax.net.ssl.*; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.KeyStore; -import java.security.SecureRandom; - -public class GeminiSSLSetUp { - private final Path certPasswordFile; - private final Path certFile; - - @Inject - public GeminiSSLSetUp( - @Named("gemini-cert-file") Path certFile, - @Named("gemini-cert-password-file") Path certPasswordFile) { - this.certFile = certFile; - this.certPasswordFile = certPasswordFile; - } - public String getCertPassword() throws IOException { - return Files.readString(certPasswordFile); - } - - private SSLContext getContext() throws Exception { - KeyStore ks = KeyStore.getInstance("JKS", "SUN"); - ks.load(Files.newInputStream(certFile), getCertPassword().toCharArray()); - - KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); - kmf.init(ks, getCertPassword().toCharArray()); - KeyManager[] keyManagers = kmf.getKeyManagers(); - - TrustManagerFactory tmf = TrustManagerFactory.getInstance("X509"); - tmf.init(ks); - TrustManager[] trustManagers = tmf.getTrustManagers(); - - var ctx = SSLContext.getInstance("TLSv1.3"); - ctx.init(keyManagers, trustManagers, new SecureRandom()); - return ctx; - } - - - public SSLServerSocketFactory getServerSocketFactory() throws Exception { - return getContext().getServerSocketFactory(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiStatusCode.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiStatusCode.java deleted file mode 100644 index b3b205eb..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiStatusCode.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.memex.gemini.io; - -public class GeminiStatusCode { - public static final int INPUT = 10; - public static final int SUCCESS = 20; - public static final int ERROR_PERMANENT = 50; - public static final int ERROR_TEMPORARY = 40; - public static final int PROXY_ERROR = 43; - public static final int REDIRECT = 30; - public static final int REDIRECT_PERMANENT = 31; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiUserException.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiUserException.java deleted file mode 100644 index 12022d56..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/io/GeminiUserException.java +++ /dev/null @@ -1,8 +0,0 @@ -package nu.marginalia.memex.gemini.io; - -/** Throw to report message to user */ -public class GeminiUserException extends RuntimeException { - public GeminiUserException(String message) { - super(message); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/BareStaticPagePlugin.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/BareStaticPagePlugin.java deleted file mode 100644 index d3a210a3..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/BareStaticPagePlugin.java +++ /dev/null @@ -1,52 +0,0 @@ -package nu.marginalia.memex.gemini.plugins; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import nu.marginalia.memex.gemini.GeminiService; -import nu.marginalia.memex.gemini.io.GeminiConnection; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; - -public class BareStaticPagePlugin implements Plugin { - - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Path geminiServerRoot; - - @Inject - public BareStaticPagePlugin(@Named("gemini-server-root") Path geminiServerRoot) { - this.geminiServerRoot = geminiServerRoot; - } - - @Override - public boolean serve(URI url, GeminiConnection connection) throws IOException { - - final Path serverPath = getServerPath(url.getPath()); - - if (!Files.isRegularFile(serverPath)) { - return false; - } - - verifyPath(geminiServerRoot, serverPath); - logger.info("Serving {}", serverPath); - - connection.respondWithFile(serverPath, FileType.match(serverPath)); - - return true; - } - - private Path getServerPath(String requestPath) { - final Path serverPath = Path.of(geminiServerRoot + requestPath); - - if (Files.isDirectory(serverPath) && Files.isRegularFile(serverPath.resolve(GeminiService.DEFAULT_FILENAME))) { - return serverPath.resolve(GeminiService.DEFAULT_FILENAME); - } - - return serverPath; - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/FileType.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/FileType.java deleted file mode 100644 index f8472f9d..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/FileType.java +++ /dev/null @@ -1,58 +0,0 @@ -package nu.marginalia.memex.gemini.plugins; - -import java.nio.file.Path; - -public enum FileType { - GMI("gmi", "text/gemini", FileIcons.DOCUMENT, false), - GEM("gem", "text/gemini", FileIcons.DOCUMENT, false), - TXT("txt", "text/plain", FileIcons.DOCUMENT, false), - MARKDOWN("md", "text/markdown", FileIcons.DOCUMENT, false), - JAVA("java", "text/java", FileIcons.JAVA, false), - PROPERTIES("properties", "text/properties", FileIcons.SETTINGS, false), - GRADLE("gradle", "text/gradle", FileIcons.SETTINGS, false), - ZIP("zip", "application/zip", FileIcons.ZIP, true), - PNG("png", "image/png", FileIcons.IMAGE, true), - JPG("jpg", "image/jpg", FileIcons.IMAGE, true), - JPEG("jpeg", "image/jpg", FileIcons.IMAGE, true), - BIN("bin", "application/binary", FileIcons.BINARY, true), - SH("sh", "text/sh", FileIcons.SETTINGS, false), - XML("xml", "text/xml", FileIcons.DOCUMENT, false), - DOCKERFILE("Dockerfile", "text/dockerfile", FileIcons.SETTINGS, false) - ; - - public static FileType match(String fileName) { - for (var type : values()) { - if (fileName.endsWith(type.suffix)) { - return type; - } - } - return BIN; - } - - public static FileType match(Path path) { - return match(path.toString()); - } - - FileType(String suffix, String mime, String icon, boolean binary) { - this.suffix = suffix; - this.mime = mime; - - this.icon = icon; - this.binary = binary; - } - public final String suffix; - public final String mime; - public final String icon; - public final boolean binary; - -} - -class FileIcons { - public static final String DOCUMENT = "🗒"; - public static final String JAVA = "♨"; - public static final String SETTINGS = "💻"; - public static final String ZIP = "🗜"; - public static final String IMAGE = "🖼"; - public static final String DIRECTORY = "🗂"; - public static final String BINARY = "📚"; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/Plugin.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/Plugin.java deleted file mode 100644 index b0ae7f0f..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/Plugin.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.memex.gemini.plugins; - -import nu.marginalia.memex.gemini.io.GeminiConnection; -import nu.marginalia.memex.gemini.io.GeminiUserException; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; - -public interface Plugin { - /** @return true if content served */ - boolean serve(URI url, GeminiConnection connection) throws IOException; - - default void verifyPath(Path root, Path p) { - if (!p.normalize().startsWith(root)) { - throw new GeminiUserException("ಠ_ಠ That path is off limits!"); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/SearchPlugin.java b/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/SearchPlugin.java deleted file mode 100644 index 95f52bd5..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/gemini/plugins/SearchPlugin.java +++ /dev/null @@ -1,78 +0,0 @@ -package nu.marginalia.memex.gemini.plugins; - -import com.google.inject.Inject; -import nu.marginalia.memex.gemini.io.GeminiConnection; -import nu.marginalia.memex.gemini.io.GeminiStatusCode; -import org.apache.http.HttpHost; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.routing.HttpRoute; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - -public class SearchPlugin implements Plugin { - private final PoolingHttpClientConnectionManager connectionManager; - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Inject - public SearchPlugin() { - - connectionManager = new PoolingHttpClientConnectionManager(); - connectionManager.setMaxTotal(200); - connectionManager.setDefaultMaxPerRoute(20); - HttpHost host = new HttpHost("https://search.marginalia.nu/"); - connectionManager.setMaxPerRoute(new HttpRoute(host), 20); - } - - @Override - public boolean serve(URI url, GeminiConnection connection) throws IOException { - var client = HttpClients.custom() - .setConnectionManager(connectionManager) - .build(); - - if (!"/search".equals(url.getPath())) { - return false; - } - - String query = url.getRawQuery(); - - if (null == query || "".equals(query)) { - logger.info("Requesting search terms"); - connection.writeStatusLine(GeminiStatusCode.INPUT, "Please enter a search query"); - } - else { - logger.info("Delegating search query '{}'", query); - - final HttpGet get = new HttpGet(createSearchUri(query)); - final byte[] binaryResponse; - - try (var rsp = client.execute(get)) { - binaryResponse = rsp.getEntity().getContent().readAllBytes(); - } - catch (IOException ex) { - logger.error("backend error", ex); - - connection.writeStatusLine(GeminiStatusCode.PROXY_ERROR, "Failed to reach backend server"); - return true; - } - - connection - .writeStatusLine(GeminiStatusCode.SUCCESS, "text/gemini") - .writeBytes(binaryResponse); - } - return true; - } - - private URI createSearchUri(String query) { - try { - return new URI("https://search.marginalia.nu/search?format=gmi&query="+query); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/Memex.java b/other/memex/src/main/java/nu/marginalia/memex/memex/Memex.java deleted file mode 100644 index 5376f86c..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/Memex.java +++ /dev/null @@ -1,244 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.memex.gemini.GeminiService; -import nu.marginalia.memex.gemini.gmi.GemtextDatabase; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.util.dithering.FloydSteinbergDither; -import nu.marginalia.memex.util.dithering.Palettes; -import nu.marginalia.memex.memex.change.GemtextTombstoneUpdateCaclulator; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexNode; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.renderer.MemexRendererers; -import nu.marginalia.memex.memex.system.MemexFileSystemMonitor; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.memex.system.git.MemexGitRepo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import javax.imageio.ImageIO; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -@Singleton -public class Memex { - - private final MemexData data; - private final MemexFileSystemMonitor monitor; - private final MemexGitRepo gitRepo; - private final MemexLoader loader; - - private final MemexFileWriter resources; - private final GemtextTombstoneUpdateCaclulator tombstoneUpdateCaclulator; - - private final FloydSteinbergDither ditherer = new FloydSteinbergDither(Palettes.MARGINALIA_PALETTE, 640, 480); - private final MemexRendererers renderers; - - private static final Logger logger = LoggerFactory.getLogger(Memex.class); - - @Inject - public Memex(MemexData data, - @Nullable MemexFileSystemMonitor monitor, - MemexGitRepo gitRepo, MemexLoader loader, - @Named("html") MemexFileWriter htmlFiles, - GemtextTombstoneUpdateCaclulator tombstoneUpdateCaclulator, - MemexRendererers renderers, - GeminiService geminiService) { - this.data = data; - this.monitor = monitor; - this.gitRepo = gitRepo; - this.loader = loader; - this.resources = htmlFiles; - this.tombstoneUpdateCaclulator = tombstoneUpdateCaclulator; - this.renderers = renderers; - - Schedulers.io().scheduleDirect(this::load); - if (monitor != null) { - Schedulers.io().schedulePeriodicallyDirect(this::refreshUpdatedUrls, 1, 1, TimeUnit.SECONDS); - } - - Schedulers.newThread().scheduleDirect(geminiService::run); - } - - private void refreshUpdatedUrls() { - var updatedUrls = monitor.getUpdatedUrls(); - for (var url : updatedUrls) { - try { - if (url.toString().endsWith(".gmi")) { - var updates = loader.reloadNode(url); - updates.forEach(renderers::render); - - if (!updates.isEmpty()) { - renderers.render(url.getParentUrl()); - } - } else if (url.toString().endsWith(".png")) { - var updates = loader.reloadImage(url); - renderers.render(url); - - if (!updates.isEmpty()) { - renderers.render(url.getParentUrl()); - } - } - - if (tombstoneUpdateCaclulator.isTombstoneFile(url)) { - loader.loadTombstones().forEach(renderers::render); - } - if (tombstoneUpdateCaclulator.isRedirectFile(url)) { - loader.loadRedirects().forEach(renderers::render); - } - } - catch (Exception ex) { - logger.error("Failed to refresh URL " + url, ex); - } - } - } - - private void load() { - copyStylesheet(); - - try { - loader.load(); - renderAll(); - } - catch (IOException ex) { - logger.error("Failed to load", ex); - } - } - - private void copyStylesheet() { - try (var resource = Objects.requireNonNull( - ClassLoader.getSystemResourceAsStream("static/memex/style-new.css"), "Could not load stylesheet")) { - resources.write(new MemexNodeUrl("/style-new.css"), resource.readAllBytes()); - } - catch (Exception ex) { - logger.error("Failed to copy stylesheet", ex); - } - - try (var resource = Objects.requireNonNull( - ClassLoader.getSystemResourceAsStream("static/memex/ico/dir.png"), "Could not copy file")) { - resources.write(new MemexNodeUrl("/ico/dir.png"), resource.readAllBytes()); - } - catch (Exception ex) { - logger.error("Failed to copy file", ex); - } - - - try (var resource = Objects.requireNonNull( - ClassLoader.getSystemResourceAsStream("static/memex/ico/file.png"), "Could not copy file")) { - resources.write(new MemexNodeUrl("/ico/file.png"), resource.readAllBytes()); - } - catch (Exception ex) { - logger.error("Failed to copy file", ex); - } - - - try (var resource = Objects.requireNonNull( - ClassLoader.getSystemResourceAsStream("static/memex/ico/root.png"), "Could not copy file")) { - resources.write(new MemexNodeUrl("/ico/root.png"), resource.readAllBytes()); - } - catch (Exception ex) { - logger.error("Failed to copy file", ex); - } - - try (var resource = Objects.requireNonNull( - ClassLoader.getSystemResourceAsStream("static/memex/ico/pic16.png"), "Could not copy file")) { - resources.write(new MemexNodeUrl("/ico/pic16.png"), resource.readAllBytes()); - } - catch (Exception ex) { - logger.error("Failed to copy file", ex); - } - } - - private void renderAll() { - data.forEach((url, doc) -> { - renderers.render(url); - }); - data.getDirectories().forEach(renderers::render); - data.getImages().forEach(img -> renderers.render(img.path)); - - data.getTombstones().ifPresent(this::renderTombstoneFromGemtextDb); - data.getRedirects().ifPresent(this::renderTombstoneFromGemtextDb); - } - - - private void renderTombstoneFromGemtextDb(GemtextDatabase db) { - db.keys() - .stream() - .map(MemexNodeUrl::new) - .filter(url -> getDocument(url) == null) - .forEach(renderers::render); - } - - public void updateNode(MemexNodeUrl node, String text) throws IOException { - var nodes = loader.updateNode(node, text); - - nodes.forEach(renderers::render); - - renderers.render(node.getParentUrl()); - } - - public GemtextDocument getDocument(MemexNodeUrl url) { - return data.getDocument(url); - } - public MemexImage getImage(MemexNodeUrl url) { - return data.getImage(url); - } - - - public void createNode(MemexNodeUrl node, String text) throws IOException { - var nodes = loader.createNode(node, text); - - nodes.forEach(renderers::render); - - renderers.render(node.getParentUrl()); - } - - - public void uploadImage(MemexNodeUrl url, byte[] bytes) throws IOException { - - var image = ImageIO.read(new ByteArrayInputStream(bytes)); - var convertedImage = ditherer.convert(image); - var baosOut = new ByteArrayOutputStream(); - ImageIO.write(convertedImage, "png", baosOut); - - loader.uploadImage(url, baosOut.toByteArray()); - - renderers.render(url); - renderers.render(url.getParentUrl()); - } - - public void delete(MemexNode node, String message) throws IOException { - tombstoneUpdateCaclulator.addTombstone(node.getUrl(), message) - .visit(this); - loader.loadTombstones(); - loader.delete(node).forEach(renderers::render); - } - - public List getDocumentsByPath(MemexNodeUrl url) { - return data.getDocumentsByPath(url); - } - - public void gitPull() { - gitRepo.pull(); - } - - public void rename(MemexNode src, MemexNodeUrl dst) throws IOException { - tombstoneUpdateCaclulator.addRedirect(src.getUrl(), dst.toString()) - .visit(this); - loader.loadRedirects(); - loader.rename(src, dst).forEach(renderers::render); - } - - public byte[] getRaw(MemexNodeUrl url) throws IOException { - return loader.getRaw(url); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexConfigurationModule.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexConfigurationModule.java deleted file mode 100644 index f0cb1bac..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexConfigurationModule.java +++ /dev/null @@ -1,87 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.inject.AbstractModule; -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; -import com.google.inject.name.Names; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.GeminiService; -import nu.marginalia.memex.gemini.GeminiServiceDummy; -import nu.marginalia.memex.gemini.GeminiServiceImpl; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.memex.system.git.MemexGitRepo; -import nu.marginalia.memex.memex.system.git.MemexGitRepoDummy; -import nu.marginalia.memex.memex.system.git.MemexGitRepoImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.file.Path; - -public class MemexConfigurationModule extends AbstractModule { - private static final Logger logger = LoggerFactory.getLogger(MemexConfigurationModule.class); - - private static final String MEMEX_ROOT_PROPERTY = System.getProperty("memex-root", "/var/lib/wmsa/memex"); - private static final String MEMEX_HTML_PROPERTY = System.getProperty("memex-html-resources", "/var/lib/wmsa/memex-html"); - private static final String MEMEX_GMI_PROPERTY = System.getProperty("memex-gmi-resources", "/var/lib/wmsa/memex-gmi"); - - private static final boolean MEMEX_DISABLE_GIT = Boolean.getBoolean("memex-disable-git"); - private static final boolean MEMEX_DISABLE_GEMINI = Boolean.getBoolean("memex-disable-gemini"); - - @SneakyThrows - public MemexConfigurationModule() { - Thread.sleep(100); - } - - public void configure() { - bind(Path.class).annotatedWith(Names.named("memex-root")).toInstance(Path.of(MEMEX_ROOT_PROPERTY)); - bind(Path.class).annotatedWith(Names.named("memex-html-resources")).toInstance(Path.of(MEMEX_HTML_PROPERTY)); - bind(Path.class).annotatedWith(Names.named("memex-gmi-resources")).toInstance(Path.of(MEMEX_GMI_PROPERTY)); - - bind(String.class).annotatedWith(Names.named("tombestone-special-file")).toInstance("/special/tombstone.gmi"); - bind(String.class).annotatedWith(Names.named("redirects-special-file")).toInstance("/special/redirect.gmi"); - - switchImpl(MemexGitRepo.class, MEMEX_DISABLE_GIT, MemexGitRepoDummy.class, MemexGitRepoImpl.class); - switchImpl(GeminiService.class, MEMEX_DISABLE_GEMINI, GeminiServiceDummy.class, GeminiServiceImpl.class); - - bind(MemexFileWriter.class).annotatedWith(Names.named("html")).toProvider(MemexHtmlWriterProvider.class); - bind(MemexFileWriter.class).annotatedWith(Names.named("gmi")).toProvider(MemexGmiWriterProvider.class); - } - - void switchImpl(Class impl, boolean param, Class ifEnabled, Class ifDisabled) { - final Class choice; - if (param) { - choice = ifEnabled; - } - else { - choice = ifDisabled; - } - bind(impl).to(choice).asEagerSingleton(); - } - - public static class MemexHtmlWriterProvider implements Provider { - private final Path path; - - @Inject - public MemexHtmlWriterProvider(@Named("memex-html-resources") Path resources) { - this.path = resources; - } - @Override - public MemexFileWriter get() { - return new MemexFileWriter(path); - } - } - - public static class MemexGmiWriterProvider implements Provider { - private final Path path; - - @Inject - public MemexGmiWriterProvider(@Named("memex-gmi-resources") Path resources) { - this.path = resources; - } - @Override - public MemexFileWriter get() { - return new MemexFileWriter(path); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexData.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexData.java deleted file mode 100644 index 42bb42ba..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexData.java +++ /dev/null @@ -1,150 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.inject.Singleton; -import nu.marginalia.memex.gemini.gmi.GemtextDatabase; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.fs.MemexFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.function.BiConsumer; - -@Singleton -public class MemexData { - private final MemexLinks links = new MemexLinks(); - private final Map documents = new HashMap<>(); - - private final Map images = new HashMap<>(); - private final MemexFileSystem fileSystem = new MemexFileSystem(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - private GemtextDatabase tombstones = null; - private GemtextDatabase redirects = null; - - public synchronized Collection getImages() { - return new ArrayList<>(images.values()); - } - public synchronized Collection getDocuments() { return new ArrayList<>(documents.values()); } - - public synchronized void setTombstones(GemtextDatabase tombstones) { - this.tombstones = tombstones; - } - public synchronized void setRedirects(GemtextDatabase redirects) { - this.redirects = redirects; - } - - public synchronized void addDocument(MemexNodeUrl url, GemtextDocument doc) { - logger.debug("addDocument({})", url); - documents.put(url, doc); - fileSystem.register(doc); - } - - public synchronized void addImage(MemexNodeUrl url, MemexImage img) { - images.put(url, img); - fileSystem.register(img); - } - - public Optional getTombstones() { - return Optional.ofNullable(tombstones); - } - public Optional getRedirects() { - return Optional.ofNullable(redirects); - } - - public synchronized void updateOutlinks(MemexNodeUrl url, GemtextDocument doc) { - - var linksForNode = new TreeSet<>(Comparator.comparing(MemexLink::getDest)); - - MemexNodeUrl srcUrl = "index.gmi".equals(url.getFilename()) ? url.getParentUrl() : url; - - for (var link : doc.getLinks()) { - link.getUrl().visitNodeUrl(nodeUrl -> - linksForNode.add(new MemexLink(nodeUrl, srcUrl, doc.getTitle(), doc.getHeadingForElement(link), link.getHeading())) - ); - } - - links.setOutlinks(srcUrl, linksForNode); - } - - public synchronized Set getNeighbors(MemexNodeUrl url) { - return links.getNeighbors(url); - } - - public synchronized void forEach(BiConsumer consumer) { - documents.forEach(consumer); - } - - public synchronized GemtextDocument getDocument(MemexNodeUrl url) { - return documents.get(url); - } - - public synchronized MemexImage getImage(MemexNodeUrl url) { - return images.get(url); - } - public synchronized List getBacklinks(MemexNodeUrl... urls) { - return links.getBacklinks(urls); - } - - public synchronized List getDocumentsByPath(MemexNodeUrl url) { - return fileSystem.getDocuments(url); - } - public synchronized List getImagesByPath(MemexNodeUrl url) { - return fileSystem.getImages(url); - } - public synchronized List getSubdirsByPath(MemexNodeUrl url) { - return fileSystem.getSubdirs(url); - } - - public MemexFileSystem getFilesystem() { - return fileSystem; - } - - public List getDirectories() { - return fileSystem.getAllDirectories(); - } - public boolean isDirectory(MemexNodeUrl url) { - return fileSystem.isDirectory(url); - } - - public synchronized Set deleteImage(MemexNodeUrl url) { - images.remove(url); - fileSystem.remove(url); - - Set affectedUrls = new HashSet<>(); - - affectedUrls.add(url); - affectedUrls.add(url.getParentUrl()); - - return affectedUrls; - } - - public synchronized Set deleteDocument(MemexNodeUrl url) { - Set affectedUrls = new HashSet<>(); - - affectedUrls.add(url); - affectedUrls.add(url.getParentUrl()); - - links.getOutlinks(url) - .stream() - .map(MemexLink::getDest) - .forEach(affectedUrls::add); - - documents.remove(url); - fileSystem.remove(url); - - links.remove(url); - - return affectedUrls; - } - - public boolean hasTombstone(MemexNodeUrl url) { - if (tombstones != null && tombstones.getLinkData(url).isPresent()) - return true; - if (redirects != null && redirects.getLinkData(url).isPresent()) - return true; - return false; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLinks.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLinks.java deleted file mode 100644 index 68168baa..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLinks.java +++ /dev/null @@ -1,54 +0,0 @@ -package nu.marginalia.memex.memex; - -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.*; -import java.util.stream.Collectors; - -public class MemexLinks { - private Map> backLinks = new HashMap<>(); - private final Map> links = new HashMap<>(); - - public void updateBacklinks() { - backLinks.clear(); - backLinks = links.values().stream() - .flatMap(Set::stream) - .collect(Collectors.groupingBy(MemexLink::getDest)); - } - - public Set getNeighbors(MemexNodeUrl url) { - final Set neighbors = new HashSet<>(); - - links.getOrDefault(url, Collections.emptySet()).stream().map(MemexLink::getDest) - .forEach(neighbors::add); - backLinks.getOrDefault(url, Collections.emptyList()).stream() - .map(MemexLink::getSrc) - .forEach(neighbors::add); - - return neighbors; - } - - public void setOutlinks(MemexNodeUrl url, TreeSet linksForNode) { - links.put(url, linksForNode); - updateBacklinks(); - } - - public List getBacklinks(MemexNodeUrl... urls) { - return Arrays.stream(urls) - .map(backLinks::get) - .filter(Objects::nonNull) - .flatMap(List::stream) - .sorted(Comparator.comparing(MemexLink::getSrc)) - .collect(Collectors.toList()); - } - - public Set getOutlinks(MemexNodeUrl url) { - return links.getOrDefault(url, Collections.emptySet()); - } - - public void remove(MemexNodeUrl url) { - links.remove(url); - updateBacklinks(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLoader.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLoader.java deleted file mode 100644 index 8945cb77..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexLoader.java +++ /dev/null @@ -1,265 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.common.collect.Sets; -import com.google.inject.Inject; -import com.google.inject.name.Named; -import nu.marginalia.memex.gemini.gmi.GemtextDatabase; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexNode; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.system.MemexFileSystemModifiedTimes; -import nu.marginalia.memex.memex.system.MemexSourceFileSystem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.CheckReturnValue; -import java.io.File; -import java.io.IOException; -import java.nio.file.*; -import java.util.*; - -public class MemexLoader { - private final MemexData data; - private final MemexFileSystemModifiedTimes modifiedTimes; - private final Path root; - private final MemexSourceFileSystem sourceFileSystem; - - private final String tombstonePath; - private final String redirectsPath; - - private static final Logger logger = LoggerFactory.getLogger(MemexLoader.class); - - @Inject - public MemexLoader(MemexData data, - MemexFileSystemModifiedTimes modifiedTimes, - MemexSourceFileSystem sourceFileSystem, - @Named("memex-root") Path root, - @Named("tombestone-special-file") String tombstonePath, - @Named("redirects-special-file") String redirectsPath) { - - this.data = data; - this.modifiedTimes = modifiedTimes; - this.sourceFileSystem = sourceFileSystem; - this.root = root; - this.tombstonePath = tombstonePath; - this.redirectsPath = redirectsPath; - } - - - public void load() throws IOException { - - loadTombstones(); - loadRedirects(); - - try (var files = Files.walk(root)) { - files.forEach(this::loadFile); - } - - data.getFilesystem().recalculateDirectories(); - - } - - private void loadFile(Path p) { - var file = p.toFile(); - - try { - if (p.toString().contains(".git")) { - return; - } - if (file.isDirectory() && !file.getName().startsWith(".")) { - data.getFilesystem().registerDir(MemexNodeUrl.ofRelativePath(root, p)); - } else if (isGemtext(file)) { - loadNode(p); - } else if (isImage(file)) { - loadImage(p); - } - } - catch (IOException ex) { - logger.error("Failed to load file " + p, ex); - } - } - - public void loadImage(Path p) throws IOException { - if (!modifiedTimes.isFreshUpdate(p)) { - return; - } - - var url = MemexNodeUrl.ofRelativePath(root, p); - data.addImage(url, new MemexImage(url, p)); - logger.info("Loading {}", p); - } - - public Set loadTombstones() { - var oldValues = data.getTombstones(); - var newValues = loadGemtextDb(Path.of(root + tombstonePath)); - - newValues.ifPresent(data::setTombstones); - - - if (newValues.isPresent()) { - if (oldValues.isPresent()) { - var oldTs = oldValues.get(); - var newTs = newValues.get(); - return oldTs.difference(newTs); - } - } - - return Collections.emptySet(); - } - - public Set loadRedirects() { - var oldValues = data.getTombstones(); - var newValues = loadGemtextDb(Path.of(root + redirectsPath)); - - newValues.ifPresent(data::setRedirects); - - if (newValues.isPresent()) { - if (oldValues.isPresent()) { - var oldTs = oldValues.get(); - var newTs = newValues.get(); - return oldTs.difference(newTs); - } - } - - return Collections.emptySet(); - } - - private Optional loadGemtextDb(Path p) { - if (Files.exists(p)) { - try { - return Optional.of(GemtextDatabase.of(MemexNodeUrl.ofRelativePath(root, p), p)); - } catch (IOException e) { - logger.error("Failed to load database " + p, e); - } - } - return Optional.empty(); - } - - private boolean isGemtext(File f) { - return f.isFile() && f.getName().endsWith(".gmi"); - } - - private boolean isImage(File f) { - return f.isFile() && f.getName().endsWith(".png"); - } - - @CheckReturnValue - public Collection updateNode(MemexNodeUrl url, String contents) throws IOException { - sourceFileSystem.replaceFile(url, contents); - return loadNode(url); - } - - @CheckReturnValue - public Collection createNode(MemexNodeUrl url, String contents) throws IOException { - sourceFileSystem.createFile(url, contents); - return loadNode(url); - } - - - public MemexImage uploadImage(MemexNodeUrl url, byte[] bytes) throws IOException { - sourceFileSystem.createFile(url, bytes); - - var img = new MemexImage(url, url.asAbsolutePath(root)); - data.addImage(url, img); - return img; - } - - - public Set reloadImage(MemexNodeUrl url) throws IOException { - var path = url.asAbsolutePath(root); - if (!Files.exists(path)) { - return data.deleteImage(url); - } - else { - loadImage(path); - Set affectedUrls = new HashSet<>(); - affectedUrls.add(url); - - for (var u = url.getParentUrl(); u != null; u = u.getParentUrl()) { - affectedUrls.add(u); - } - - return affectedUrls; - } - } - - public Set reloadNode(MemexNodeUrl url) throws IOException { - var path = url.asAbsolutePath(root); - if (!Files.exists(path)) { - return data.deleteDocument(url); - } - else { - return loadNode(path); - } - } - - public Set loadNode(Path path) throws IOException { - - if (!modifiedTimes.isFreshUpdate(path)) { - return Set.of(MemexNodeUrl.ofRelativePath(root, path)); - } - - logger.info("Loading {}", path); - - return loadNode(MemexNodeUrl.ofRelativePath(root, path)); - } - - public Set loadNode(MemexNodeUrl url) throws IOException { - - var doc = GemtextDocument.of(url, url.asAbsolutePath(root)); - - data.addDocument(url, doc); - - Set urlsAffected = data.getNeighbors(url); - - data.updateOutlinks(url, doc); - - urlsAffected.addAll(data.getNeighbors(url)); - urlsAffected.add(url); - urlsAffected.removeIf(u -> null == data.getDocument(u)); - - for (var u = url.getParentUrl(); u != null; u = u.getParentUrl()) { - urlsAffected.add(u); - } - - return urlsAffected; - } - - public Set delete(MemexNode node) throws IOException { - sourceFileSystem.delete(node.getUrl()); - return node.visit(new MemexNode.MemexNodeVisitor<>() { - @Override - public Set onDocument(MemexNodeUrl url) { - return data.deleteDocument(url); - } - - @Override - public Set onImage(MemexNodeUrl url) { - return data.deleteImage(url); - } - }); - } - - public Set rename(MemexNode src, MemexNodeUrl dst) throws IOException { - sourceFileSystem.renameFile(src.getUrl(), dst); - return src.visit(new MemexNode.MemexNodeVisitor>() { - @Override - public Set onDocument(MemexNodeUrl url) throws IOException { - var changes = data.deleteDocument(url); - return Sets.union(changes, reloadNode(dst)); - } - - @Override - public Set onImage(MemexNodeUrl url) throws IOException { - var changes = data.deleteImage(url); - return Sets.union(changes, reloadImage(dst)); - } - }); - - } - - public byte[] getRaw(MemexNodeUrl url) throws IOException { - return sourceFileSystem.getRaw(url); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexMain.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexMain.java deleted file mode 100644 index c5601380..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexMain.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.memex.MemexServiceDescriptors; -import nu.marginalia.memex.gemini.GeminiConfigurationModule; -import nu.marginalia.service.MainClass; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.service.module.ConfigurationModule; -import nu.marginalia.service.server.Initialization; - -public class MemexMain extends MainClass { - private final MemexService service; - - @Inject - public MemexMain(MemexService service) { - this.service = service; - } - - public static void main(String... args) { - MainClass.init(ServiceId.Other_Memex, args); - - Injector injector = Guice.createInjector( - new MemexConfigurationModule(), - new GeminiConfigurationModule(), - new ConfigurationModule(MemexServiceDescriptors.descriptors, ServiceId.Other_Memex)); - injector.getInstance(MemexMain.class); - injector.getInstance(Initialization.class).setReady(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexService.java b/other/memex/src/main/java/nu/marginalia/memex/memex/MemexService.java deleted file mode 100644 index 5e799298..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/MemexService.java +++ /dev/null @@ -1,292 +0,0 @@ -package nu.marginalia.memex.memex; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import lombok.SneakyThrows; -import nu.marginalia.client.Context; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.auth.client.AuthClient; -import nu.marginalia.memex.memex.model.render.*; -import nu.marginalia.memex.memex.change.GemtextMutation; -import nu.marginalia.memex.memex.change.update.GemtextDocumentUpdateCalculator; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.MetricsServer; -import nu.marginalia.service.server.Service; -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; -import spark.Spark; - -import javax.servlet.MultipartConfigElement; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.Objects; - -import static spark.Spark.*; - -public class MemexService extends Service { - private final GemtextDocumentUpdateCalculator updateCalculator; - private final Memex memex; - private final MemexHtmlRenderer renderer; - private final AuthClient authClient; - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Inject - public MemexService(@Named("service-host") String ip, - @Named("service-port") Integer port, - GemtextDocumentUpdateCalculator updateCalculator, - Memex memex, - MemexHtmlRenderer renderer, - AuthClient authClient, - Initialization initialization, - MetricsServer metricsServer, - @Named("memex-html-resources") Path memexHtmlDir - ) { - - super(ip, port, initialization, metricsServer, () -> { - staticFiles.externalLocation(memexHtmlDir.toString()); - staticFiles.disableMimeTypeGuessing(); - staticFiles.registerMimeType("gmi", "text/html"); - staticFiles.registerMimeType("png", "text/html"); - staticFiles.expireTime(60); - staticFiles.header("Cache-control", "public,proxy-revalidate"); - }); - - this.updateCalculator = updateCalculator; - this.memex = memex; - this.renderer = renderer; - this.authClient = authClient; - - Spark.get("git-pull", this::gitPull); - - Spark.path("public/api", () -> { - before((req, rsp) -> { - logger.info("{} {}", req.requestMethod(), req.pathInfo()); - }); - after((req, rsp) -> { - rsp.header("Cache-control", "no-cache"); - }); - - post("/create", this::create); - get("/create", this::createForm, this::renderModel); - post("/upload", this::upload); - get("/upload", this::uploadForm, this::renderModel); - post("/update", this::update); - get("/update", this::updateForm, this::renderModel); - post("/rename", this::rename); - get("/rename", this::renameForm, this::renderModel); - post("/delete", this::delete); - get("/delete", this::deleteForm, this::renderModel); - - get("/raw", this::raw); - }); - } - - private Object raw(Request request, Response response) throws IOException { - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - - response.type(url.toNode().getType().mime); - response.header("Content-Disposition", "attachment; filename=" + url.getFilename()); - response.raw().getOutputStream().write(memex.getRaw(url)); - - return ""; - } - - private Object renameForm(Request request, Response response) { - final String type = Objects.requireNonNull(request.queryParams("type")); - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - if ("gmi".equals(type)) { - var doc = memex.getDocument(url); - if (null == doc) { - Spark.halt(404); - } - - final String docHtml = doc.render(new GemtextRendererFactory("", url.toString()).htmlRendererEditable()); - return new MemexRendererRenameFormModel(docHtml, - null, url, "gmi"); - } - else if ("img".equals(type)) { - var img = memex.getImage(url); - if (null == img) { - Spark.halt(404); - } - return new MemexRendererRenameFormModel(null, - new MemexRendererImageModel(img, Collections.emptyList(), null), - url, "img"); - } - - Spark.halt(HttpStatus.SC_BAD_REQUEST); - return null; - } - - private Object rename(Request request, Response response) throws IOException { - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - var url = Objects.requireNonNull(request.queryParams("url")); - var name = Objects.requireNonNull(request.queryParams("name")); - var type = Objects.requireNonNull(request.queryParams("type")); - var confirm = Objects.requireNonNull(request.queryParams("confirm")); - - if (!"on".equals(confirm)) { - logger.error("Confirm dialog not checked, was {}", confirm); - Spark.halt(HttpStatus.SC_BAD_REQUEST, "Confirm was not checked"); - } - - memex.rename(new MemexNodeUrl(url).toNode(), new MemexNodeUrl(name)); - - response.redirect("https://memex.marginalia.nu/"+name); - return null; - - } - - private Object gitPull(Request request, Response response) { - logger.info("Git pull by request"); - memex.gitPull(); - return "Ok"; - } - - private String renderModel(Object model) { - return ((MemexRendererableDirect)model).render(renderer); - } - - private MemexRendererDeleteFormModel deleteForm(Request request, Response response) { - final String type = Objects.requireNonNull(request.queryParams("type")); - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - if ("gmi".equals(type)) { - var doc = memex.getDocument(url); - if (null == doc) { - Spark.halt(404); - } - - final String docHtml = doc.render(new GemtextRendererFactory("", url.toString()).htmlRendererEditable()); - return new MemexRendererDeleteFormModel(docHtml, - null, url, "gmi"); - } - else if ("img".equals(type)) { - var img = memex.getImage(url); - if (null == img) { - Spark.halt(404); - } - return new MemexRendererDeleteFormModel(null, - new MemexRendererImageModel(img, Collections.emptyList(), null), - url, "img"); - } - - Spark.halt(HttpStatus.SC_BAD_REQUEST); - return null; - } - - private Object delete(Request request, Response response) throws IOException { - authClient.requireLogIn(Context.fromRequest(request)); - - var url = Objects.requireNonNull(request.queryParams("url")); - var message = Objects.requireNonNull(request.queryParams("note")); - var type = Objects.requireNonNull(request.queryParams("type")); - var confirm = Objects.requireNonNull(request.queryParams("confirm")); - - if (!"on".equals(confirm)) { - logger.error("Confirm dialog not checked, was {}", confirm); - Spark.halt(HttpStatus.SC_BAD_REQUEST, "Confirm was not checked"); - } - - memex.delete(new MemexNodeUrl(url).toNode(), message); - - response.redirect("https://memex.marginalia.nu/"+url); - return null; - } - - private Object update(Request request, Response response) throws IOException { - authClient.requireLogIn(Context.fromRequest(request)); - - String extUrl = Objects.requireNonNull(request.queryParams("url")); - String extSection = Objects.requireNonNull(request.queryParams("section")); - String newSectionText = Objects.requireNonNull(request.queryParams("text")); - - var url = new MemexNodeUrl(extUrl); - var section = MemexNodeHeadingId.parse(extSection); - var lines = Arrays.asList(newSectionText.split("\r?\n")).toArray(String[]:: new); - - var sectionGemtext = new GemtextDocument(url, lines, section); - var updates = updateCalculator.calculateUpdates(memex.getDocument(url), section, sectionGemtext); - - for (GemtextMutation mutation : updates) { - mutation.visit(memex); - } - - response.redirect("https://memex.marginalia.nu/"+extUrl); - return ""; - } - - private Object create(Request request, Response response) throws IOException { - authClient.requireLogIn(Context.fromRequest(request)); - - String directory = Objects.requireNonNull(request.queryParams("directory")); - String filename = Objects.requireNonNull(request.queryParams("filename")); - String text = Objects.requireNonNull(request.queryParams("text")); - var url = new MemexNodeUrl(Path.of(directory).resolve(filename).toString()); - - memex.createNode(url, text); - - response.redirect("https://memex.marginalia.nu/"+directory + "/" + filename); - return ""; - } - - private Object createForm(Request request, Response response) { - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - return new MemexRenderCreateFormModel(url, memex.getDocumentsByPath(url)); - } - - private Object uploadForm(Request request, Response response) { - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - return new MemexRenderUploadFormModel(url, memex.getDocumentsByPath(url)); - } - - private Object updateForm(Request request, Response response) { - final MemexNodeUrl url = new MemexNodeUrl(Objects.requireNonNull(request.queryParams("url"))); - authClient.redirectToLoginIfUnauthenticated("MEMEX", request, response); - - var doc = memex.getDocument(url); - - return new MemexRenderUpdateFormModel(url, doc.getTitle(), "0", doc.getSectionGemtext(MemexNodeHeadingId.ROOT)); - } - - - @SneakyThrows - private Object upload(Request request, Response response) { - authClient.requireLogIn(Context.fromRequest(request)); - - request.attribute("org.eclipse.jetty.multipartConfig", new MultipartConfigElement("/temp", 50*1024*1024, 50*1024*1024, 25*1024*1024)); - - String directory = Objects.requireNonNull(request.queryParams("directory")); - String filename = Objects.requireNonNull(request.queryParams("filename")); - var url = new MemexNodeUrl(Path.of(directory).resolve(filename).toString()); - try (InputStream input = request.raw().getPart("file").getInputStream()) { - byte[] data = input.readAllBytes(); - memex.uploadImage(url, data); - } - - response.redirect("https://memex.marginalia.nu/"+directory + "/" + filename); - return ""; - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextAppend.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextAppend.java deleted file mode 100644 index 8f4e0b6e..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextAppend.java +++ /dev/null @@ -1,70 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import lombok.AllArgsConstructor; -import lombok.ToString; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.io.IOException; - -@AllArgsConstructor @ToString -public class GemtextAppend implements GemtextMutation { - public final MemexNodeUrl doc; - public final MemexNodeHeadingId id; - public final String[] lines; - - @Override - public void visit(Memex memex) throws IOException { - memex.updateNode(doc, calculateAppend(memex.getDocument(doc))); - } - - public String calculateAppend(GemtextDocument document) { - - StringBuilder result = new StringBuilder(); - var renderer = new GemtextRendererFactory().gemtextRendererAsIs(); - - var lines = document.getLines(); - - int i = 0; - // Copy from before heading - for (; i < lines.length; i++) { - var item = lines[i]; - - if (item.getHeading().isChildOf(id)) { - break; - } - else { - result.append(item.visit(renderer)).append('\n'); - } - } - - // Copy contents of heading - for (; i < lines.length; i++) { - var item = lines[i]; - - if (!item.getHeading().isChildOf(id)) { - break; - } - else { - result.append(item.visit(renderer)).append('\n'); - } - } - - // Insert new lines - for (String newLine : this.lines) { - result.append(newLine).append('\n'); - } - - // Copy contents from after heading - for (;i < lines.length; i++) { - var item = lines[i]; - result.append(item.visit(renderer)).append('\n'); - } - - return result.toString(); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreate.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreate.java deleted file mode 100644 index f44c3875..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreate.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import lombok.AllArgsConstructor; -import lombok.ToString; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.io.IOException; - -@AllArgsConstructor @ToString -public class GemtextCreate implements GemtextMutation { - public final MemexNodeUrl doc; - public final String text; - - @Override - public void visit(Memex memex) throws IOException { - memex.createNode(doc, text); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreateOrMutate.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreateOrMutate.java deleted file mode 100644 index 7f60a759..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextCreateOrMutate.java +++ /dev/null @@ -1,26 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import lombok.AllArgsConstructor; -import lombok.ToString; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.io.IOException; - -@AllArgsConstructor @ToString -public class GemtextCreateOrMutate implements GemtextMutation { - public final MemexNodeUrl doc; - public final String text; - public final GemtextMutation mutation; - - @Override - public void visit(Memex memex) throws IOException { - if (memex.getDocument(doc) == null) { - memex.createNode(doc, text); - } - if (memex.getDocument(doc) == null) - throw new IllegalStateException(); - - mutation.visit(memex); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextMutation.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextMutation.java deleted file mode 100644 index 7612b1a0..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextMutation.java +++ /dev/null @@ -1,18 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.io.IOException; - -public interface GemtextMutation { - void visit(Memex memex) throws IOException; - - static GemtextMutation createOrAppend(MemexNodeUrl url, String template, MemexNodeHeadingId heading, String... lines) { - return new GemtextCreateOrMutate(url, template, new GemtextAppend(url, heading, lines)); - } - static GemtextMutation createOrPrepend(MemexNodeUrl url, String template, MemexNodeHeadingId heading, String... lines) { - return new GemtextCreateOrMutate(url, template, new GemtextPrepend(url, heading, lines)); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextPrepend.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextPrepend.java deleted file mode 100644 index 84f70ca8..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextPrepend.java +++ /dev/null @@ -1,63 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import lombok.AllArgsConstructor; -import lombok.ToString; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -@AllArgsConstructor @ToString -public class GemtextPrepend implements GemtextMutation { - public final MemexNodeUrl doc; - public final MemexNodeHeadingId id; - public final String[] lines; - - private static final Logger logger = LoggerFactory.getLogger(GemtextPrepend.class); - - @Override - public void visit(Memex memex) throws IOException { - memex.updateNode(doc, calculatePrepend(memex.getDocument(doc))); - } - - public String calculatePrepend(GemtextDocument document) { - StringBuilder result = new StringBuilder(); - var renderer = new GemtextRendererFactory().gemtextRendererAsIs(); - var lines = document.getLines(); - int i = 0; - for (; i < lines.length; i++) { - var item = lines[i]; - - if (item.getHeading().isChildOf(id)) { - if (!id.equals(MemexNodeHeadingId.ROOT)) { - result.append(item.visit(renderer)).append('\n'); - i++; - } - break; - } - else { - result.append(item.visit(renderer)).append('\n'); - } - } - - if (i == lines.length) { - logger.warn("Heading not found in prepending heading {} of {}, falling back to append-like behavior", - id, document.getUrl()); - } - for (String newLine : this.lines) { - result.append(newLine).append('\n'); - } - - for (;i < lines.length; i++) { - var item = lines[i]; - result.append(item.visit(renderer)).append('\n'); - } - - return result.toString(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextReplace.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextReplace.java deleted file mode 100644 index 6e437fa2..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextReplace.java +++ /dev/null @@ -1,65 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import lombok.AllArgsConstructor; -import lombok.ToString; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -@AllArgsConstructor @ToString -public class GemtextReplace implements GemtextMutation { - public final MemexNodeUrl doc; - public final MemexNodeHeadingId id; - public final String[] lines; - - private static final Logger logger = LoggerFactory.getLogger(GemtextPrepend.class); - - @Override - public void visit(Memex memex) throws IOException { - memex.updateNode(doc, calculateReplace(memex.getDocument(doc))); - } - - public String calculateReplace(GemtextDocument document) { - StringBuilder result = new StringBuilder(); - var renderer = new GemtextRendererFactory().gemtextRendererAsIs(); - - var lines = document.getLines(); - int i = 0; - for (; i < lines.length; i++) { - var item = lines[i]; - - if (item.getHeading().isChildOf(id)) { - break; - } - else { - result.append(item.visit(renderer)).append('\n'); - } - } - - if (i == lines.length) { - logger.error("Heading not found in replacing heading {} of {}, writing change-data to file", - id, document.getUrl()); - result.append("# Error! Replace failed!\n"); - } - - for (;i < lines.length && lines[i].getHeading().isChildOf(id); i++) { - } - - for (String newLine : this.lines) { - result.append(newLine).append('\n'); - } - - for (;i < lines.length; i++) { - var item = lines[i]; - result.append(item.visit(renderer)).append('\n'); - } - - return result.toString(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulator.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulator.java deleted file mode 100644 index 83e03b31..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulator.java +++ /dev/null @@ -1,48 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import com.google.inject.name.Named; - -import java.util.Objects; - -@Singleton -public class GemtextTombstoneUpdateCaclulator { - private final String tombstonePath; - private final String redirectsPath; - - @Inject - public GemtextTombstoneUpdateCaclulator(@Named("tombestone-special-file") String tombstonePath, - @Named("redirects-special-file") String redirectsPath) { - - this.tombstonePath = tombstonePath; - this.redirectsPath = redirectsPath; - } - - public boolean isTombstoneFile(MemexNodeUrl url) { - return Objects.equals(url, new MemexNodeUrl(tombstonePath)); - } - public boolean isRedirectFile(MemexNodeUrl url) { - return Objects.equals(url, new MemexNodeUrl(redirectsPath)); - } - - public GemtextMutation addTombstone(MemexNodeUrl url, String message) { - var tombstoneUrl = new MemexNodeUrl(tombstonePath); - - return new GemtextCreateOrMutate(tombstoneUrl, "# Tombstones", - new GemtextAppend(tombstoneUrl, new MemexNodeHeadingId(0), - new String[] { String.format("=> %s\t%s", url, message)})); - } - - public GemtextMutation addRedirect(MemexNodeUrl url, String message) { - var redirectsUrl = new MemexNodeUrl(redirectsPath); - - return new GemtextCreateOrMutate(redirectsUrl, "# Redirects", - new GemtextAppend(redirectsUrl, new MemexNodeHeadingId(0), - new String[] { String.format("=> %s\t%s", url, message)})); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextDocumentUpdateCalculator.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextDocumentUpdateCalculator.java deleted file mode 100644 index 237ad448..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextDocumentUpdateCalculator.java +++ /dev/null @@ -1,109 +0,0 @@ -package nu.marginalia.memex.memex.change.update; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.memex.gemini.gmi.line.GemtextText; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextHeading; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRenderer; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.change.*; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -@Singleton -public class GemtextDocumentUpdateCalculator { - private final GemtextRenderer rawRenderer = new GemtextRendererFactory().gemtextRendererAsIs(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Memex memex; - - @Inject - public GemtextDocumentUpdateCalculator(Memex memex) { - this.memex = memex; - } - - public List calculateUpdates(GemtextDocument original, - MemexNodeHeadingId destId, - GemtextDocument newSection) - { - - var rewrite = new GemtextTasksRewrite(memex, original, destId, newSection); - var lines = newSection.getLines(); - - for (int i = 0; i < lines.length; i = Math.max(i+1, rewrite.processLine(lines, i))); - - List updates = new ArrayList<>(); - - updates.addAll(createKeepUpdates(original, destId, rewrite)); - updates.addAll(createDoneUpdates(original, destId, rewrite)); - updates.addAll(createTodoUpdates(original, rewrite)); - - return updates; - } - - private Collection createTodoUpdates(GemtextDocument original, GemtextTasksRewrite rewrite) { - if (!rewrite.getPushToTodo().isEmpty()) { - var doneDoc = original.getUrl().sibling("todo.gmi"); - - var update = createTodoAction(rewrite.getPushToTodo(), doneDoc); - return List.of(update); - } - return Collections.emptyList(); - } - - private Collection createDoneUpdates(GemtextDocument original, MemexNodeHeadingId destId, GemtextTasksRewrite rewrite) { - if (!rewrite.getPushToDone().isEmpty()) { - - var doneDocUrl = original.getUrl().sibling("done.gmi"); - final String doneHeadingName = rewrite.getTodaysDoneHeadingName(); - - var newDestId = - Optional.ofNullable(memex.getDocument(doneDocUrl)) - .flatMap(dest -> dest.getHeadingByName(MemexNodeHeadingId.ROOT, doneHeadingName)); - - if (newDestId.isEmpty()) { - rewrite.getPushToDone().addAll(0, - List.of(new GemtextText("", MemexNodeHeadingId.ROOT), - new GemtextHeading(new MemexNodeHeadingId(1,1), doneHeadingName, destId)) - ); - } - - var update = createDoneAction(rewrite.getPushToDone(), doneDocUrl, newDestId.orElse(new MemexNodeHeadingId(1))); - return List.of(update); - } - return Collections.emptyList(); - } - - private Collection createKeepUpdates(GemtextDocument original, MemexNodeHeadingId destId, GemtextTasksRewrite rewrite) { - if (!rewrite.getKeep().isEmpty()) { - return List.of(new GemtextReplace(original.getUrl(), destId, rewrite.getKeep().stream().map(rawRenderer::renderLine).toArray(String[]::new))); - } - return Collections.emptyList(); - } - - - @NotNull - private GemtextCreateOrMutate createDoneAction(List pushToDone, MemexNodeUrl doneDoc, MemexNodeHeadingId newDestId) { - return new GemtextCreateOrMutate( - doneDoc, "%%% TASKS\n# Done", - new GemtextPrepend(doneDoc, newDestId, pushToDone.stream().map(rawRenderer::renderLine).toArray(String[]::new)) - ); - } - - @NotNull - private GemtextCreateOrMutate createTodoAction(List pushToTodo, MemexNodeUrl doneDoc) { - return new GemtextCreateOrMutate( - doneDoc, "%%% TASKS\n# Todo", - new GemtextAppend(doneDoc, new MemexNodeHeadingId(1), pushToTodo.stream().map(rawRenderer::renderLine).toArray(String[]::new)) - ); - } - -} - diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTaskExtractor.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTaskExtractor.java deleted file mode 100644 index f85902a0..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTaskExtractor.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.memex.memex.change.update; - -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextTask; - -import java.util.List; - -class GemtextTaskExtractor { - - public static int extractTask(List dest, AbstractGemtextLine[] lines, int i) { - var taskId = ((GemtextTask) lines[i]).getId(); - - int j; - for (j = i; j < lines.length; j++) { - var item = lines[j]; - if (item.mapTask(GemtextTask::getId).map(id -> id.isChildOf(taskId)).orElse(false)) { - dest.add(item); - } - else if (!item.breaksTask()) { - dest.add(item); - } - else { - break; - } - } - if (j < lines.length) { - return Math.max(i+1, j-1); - } - return lines.length; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTasksRewrite.java b/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTasksRewrite.java deleted file mode 100644 index 0db13d26..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/change/update/GemtextTasksRewrite.java +++ /dev/null @@ -1,100 +0,0 @@ -package nu.marginalia.memex.memex.change.update; - -import lombok.Getter; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.line.AbstractGemtextLine; -import nu.marginalia.memex.gemini.gmi.line.GemtextTask; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import org.jetbrains.annotations.NotNull; - -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.function.Predicate; - -@Getter -class GemtextTasksRewrite { - private final List keep = new ArrayList<>(); - private final List pushToTodo = new ArrayList<>(); - private final List pushToDone = new ArrayList<>(); - - private final String rootHeadingName = "Done"; - private final String todoHeadingName = "Todo"; - private final String backlogHeadingName = "Backlog"; - - private final boolean isDestTodo; - private final boolean isDestDone; - private final Memex memex; - private final GemtextDocument original; - private final MemexNodeHeadingId destId; - private final GemtextDocument newSection; - - GemtextTasksRewrite(Memex memex, GemtextDocument original, MemexNodeHeadingId destId, GemtextDocument newSection) { - this.memex = memex; - this.original = original; - this.destId = destId; - this.newSection = newSection; - - - isDestTodo = isDestTodo(original, destId); - isDestDone = isDestDone(original, destId); - } - - public int processLine(AbstractGemtextLine[] lines, int i) { - var line = lines[i]; - - if (!line.mapTask(GemtextTask::getLevel).map(level -> 1 == level).orElse(false)) { - keep.add(line); - return i + 1; - } - - // It's a task - - boolean isTaskDone = line.mapTask(GemtextTask::getState).map(state -> state.done).orElse(false); - boolean isChangeDestDone = matchHeadingHierarchy(newSection, line.getHeading(), heading -> heading.contains(rootHeadingName)); - boolean isChangeDestTodo = isDestTodo(newSection, line.getHeading()); - - if (isTaskDone && !isDestDone && !isChangeDestDone) { - return GemtextTaskExtractor.extractTask(pushToDone, lines, i); - } else if (!isTaskDone && !isDestTodo && !isChangeDestTodo) { - return GemtextTaskExtractor.extractTask(pushToTodo, lines, i); - } - - keep.add(line); - return i + 1; - } - - private boolean isDestDone(GemtextDocument original, MemexNodeHeadingId destId) { - - final String currentHeadingName = getTodaysDoneHeadingName(); - - return matchHeadingHierarchy(original, destId, heading -> heading.contains(currentHeadingName)) - || matchHeadingHierarchy(original, destId, heading -> heading.contains(rootHeadingName)); - } - - @NotNull - public String getTodaysDoneHeadingName() { - return "Done " + LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE); - } - - private boolean isDestTodo(GemtextDocument original, MemexNodeHeadingId destId) { - return matchHeadingHierarchy(original, destId, heading -> heading.contains(todoHeadingName)) - || matchHeadingHierarchy(original, destId, heading -> heading.contains(backlogHeadingName)); - } - - - boolean matchHeadingHierarchy(GemtextDocument doc, MemexNodeHeadingId heading, Predicate p) { - - for (; !heading.equals(MemexNodeHeadingId.ROOT); heading = heading.parent()) { - var maybeTitle = doc.getHeading(heading); - if (maybeTitle.map(p::test).orElse(false)) { - return true; - } - - } - return false; - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSection.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSection.java deleted file mode 100644 index c2237841..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSection.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -@AllArgsConstructor @Getter -public class GemtextSection { - public final MemexNodeHeadingId id; - public final GemtextSectionAction action; - public final String[] lines; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSectionAction.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSectionAction.java deleted file mode 100644 index a55e4546..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/GemtextSectionAction.java +++ /dev/null @@ -1,6 +0,0 @@ -package nu.marginalia.memex.memex.model; - -public enum GemtextSectionAction { - REPLACE, - APPEND -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexExternalUrl.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexExternalUrl.java deleted file mode 100644 index 28784a10..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexExternalUrl.java +++ /dev/null @@ -1,18 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.EqualsAndHashCode; -import lombok.Getter; - -import java.util.Optional; - -@AllArgsConstructor @Getter @EqualsAndHashCode -public class MemexExternalUrl implements MemexUrl { - public final String url; - - public String toString() { - return url; - } - @Override - public Optional getExternUrl() { return Optional.of(this); } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexImage.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexImage.java deleted file mode 100644 index d27a5e65..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexImage.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.nio.file.Path; - -@AllArgsConstructor @Getter -public class MemexImage { - public final MemexNodeUrl path; - public final Path realPath; - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexIndexTask.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexIndexTask.java deleted file mode 100644 index 893b5854..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexIndexTask.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; - -@AllArgsConstructor @ToString @Getter -public class MemexIndexTask { - public final String task; - public final String taskId; - public final String url; - public final String type; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexLink.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexLink.java deleted file mode 100644 index c03822ac..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexLink.java +++ /dev/null @@ -1,26 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.Objects; - -@Getter @AllArgsConstructor -public class MemexLink { - public final MemexNodeUrl dest; - public final MemexNodeUrl src; - public final String title; - public final String section; - public final MemexNodeHeadingId sectionId; - - public final MemexNodeUrl getUrl() { - return src; - } - - public String getDescription() { - if (Objects.equals(title, section)) { - return title; - } - return title + " - " + section; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNode.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNode.java deleted file mode 100644 index 0730ebfe..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNode.java +++ /dev/null @@ -1,40 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.SneakyThrows; - -@AllArgsConstructor @Getter -public class MemexNode { - private final MemexNodeUrl url; - - public MemexNodeType getType() { - var fn = url.getFilename(); - if (fn.endsWith(".gmi")) { - return MemexNodeType.DOCUMENT; - } - else if (fn.endsWith(".png")) { - return MemexNodeType.IMAGE; - } - else if (fn.endsWith(".txt")) { - return MemexNodeType.TEXT; - } - else if (fn.contains(".")) { - return MemexNodeType.OTHER; - } - return MemexNodeType.DIRECTORY; - } - - @SneakyThrows - public T visit(MemexNodeVisitor visitor) { - return switch (getType()) { - case DOCUMENT -> visitor.onDocument(url); - case IMAGE -> visitor.onImage(url); - default -> null; - }; - } - public interface MemexNodeVisitor { - T onDocument(MemexNodeUrl url) throws Exception; - T onImage(MemexNodeUrl url) throws Exception; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeHeadingId.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeHeadingId.java deleted file mode 100644 index d5db6ffa..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeHeadingId.java +++ /dev/null @@ -1,74 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.EqualsAndHashCode; - -import java.util.Arrays; -import java.util.stream.Collectors; - -@EqualsAndHashCode -public class MemexNodeHeadingId implements Comparable { - private final int[] ids; - - public static final MemexNodeHeadingId ROOT = new MemexNodeHeadingId(0); - - public MemexNodeHeadingId(int... ids) { - this.ids = ids; - } - - public static MemexNodeHeadingId parse(String section) { - return new MemexNodeHeadingId(Arrays.stream(section.split("\\.")).mapToInt(Integer::parseInt).toArray()); - } - - public int getLevel() { - return ids.length; - } - - public int[] getIds() { - return ids; - } - public boolean isChildOf(MemexNodeHeadingId other) { - if (other.equals(ROOT)) { - return true; - } - if (other.ids.length > ids.length) { - return false; - } - - for (int i = 0; i < other.ids.length; i++) { - if (other.ids[i] != ids[i]) { - return false; - } - } - - return true; - } - - // This does not have the same semantics as Arrays$compare - - public int compareTo(MemexNodeHeadingId other) { - for (int i = 0; i < Math.min(ids.length, other.ids.length); i++) { - if (other.ids[i] != ids[i]) { - return ids[i] - other.ids[i]; - } - } - - return other.ids.length - ids.length; - } - - public MemexNodeHeadingId parent() { - if (ids.length <= 1) - return ROOT; - else return new MemexNodeHeadingId(Arrays.copyOfRange(ids, 0, ids.length-1)); - - } - public MemexNodeHeadingId next(int level) { - int[] newIds = Arrays.copyOf(ids, level+1); - newIds[level]++; - return new MemexNodeHeadingId(newIds); - } - - @Override - public String toString() { - return Arrays.stream(ids).mapToObj(Integer::toString).collect(Collectors.joining(".")); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeTaskId.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeTaskId.java deleted file mode 100644 index 4fd45946..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeTaskId.java +++ /dev/null @@ -1,66 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.EqualsAndHashCode; - -import java.util.Arrays; -import java.util.stream.Collectors; - -@EqualsAndHashCode -public class MemexNodeTaskId implements Comparable { - private final int[] ids; - - public MemexNodeTaskId(int... ids) { - this.ids = ids; - } - - public static MemexNodeTaskId parse(String section) { - return new MemexNodeTaskId(Arrays.stream(section.split("\\.")).mapToInt(Integer::parseInt).toArray()); - } - - public int level() { - return ids.length; - } - - public boolean isChildOf(MemexNodeTaskId other) { - if (other.ids.length > ids.length) { - return false; - } - - for (int i = 0; i < other.ids.length; i++) { - if (other.ids[i] != ids[i]) { - return false; - } - } - - return true; - } - - // This does not have the same semantics as Arrays$compare - - public int compareTo(MemexNodeTaskId other) { - for (int i = 0; i < Math.min(ids.length, other.ids.length); i++) { - if (other.ids[i] != ids[i]) { - return ids[i] - other.ids[i]; - } - } - - return other.ids.length - ids.length; - } - - public MemexNodeTaskId parent() { - if (ids.length <= 1) - return new MemexNodeTaskId(0); - else return new MemexNodeTaskId(Arrays.copyOfRange(ids, 0, ids.length-1)); - - } - public MemexNodeTaskId next(int level) { - int[] newIds = Arrays.copyOf(ids, level+1); - newIds[level]++; - return new MemexNodeTaskId(newIds); - } - - @Override - public String toString() { - return Arrays.stream(ids).mapToObj(Integer::toString).collect(Collectors.joining(".")); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeType.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeType.java deleted file mode 100644 index 33bdbb5f..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeType.java +++ /dev/null @@ -1,14 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public enum MemexNodeType { - DOCUMENT("text/gemini"), - IMAGE("image/png"), - DIRECTORY("other/directory"), - TEXT("text/plain"), - OTHER("application/binary"); - - public String mime; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeUrl.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeUrl.java deleted file mode 100644 index c919b511..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexNodeUrl.java +++ /dev/null @@ -1,98 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import org.jetbrains.annotations.NotNull; - -import java.io.File; -import java.nio.file.Path; -import java.util.Optional; -import java.util.function.Consumer; - -@Getter @EqualsAndHashCode -public class MemexNodeUrl implements MemexUrl, Comparable { - private final String url; - - public MemexNodeUrl(String url) { - if (url.startsWith("//")) { - this.url = url.substring(1); - } else { - this.url = url; - } - } - public static MemexNodeUrl ofRelativePath(Path root, Path relative) { - Path path; - - if (relative.startsWith("/")) { - path = root.relativize(relative); - } - else { - path = relative; - } - - if (File.separatorChar == '\\') - return new MemexNodeUrl("/" + path.toString().replace('\\', '/')); - return new MemexNodeUrl("/" + path); - } - - public String toString() { - return url; - } - - public String getParentStr() { - var path = asRelativePath().getParent(); - if (path == null) { - return null; - } - return path.toString(); - } - public MemexNodeUrl getParentUrl() { - var str = getParentStr(); - if (str == null) { - return null; - } - return new MemexNodeUrl(str); - } - public MemexNodeUrl sibling(String name) { - return new MemexNodeUrl(asRelativePath().resolveSibling(name).toString()); - } - public MemexNodeUrl child(String name) { - return new MemexNodeUrl(asRelativePath().resolve(name).toString()); - } - - public Path asRelativePath() { - return Path.of(url); - } - - public Path asAbsolutePath(Path root) { - Path p = Path.of(root + url); - if (p.toString().contains(".git")) { - throw new IllegalStateException(url + " touched .git"); - } - if (!p.normalize().startsWith(root)) { - throw new IllegalStateException(url + " escaped Memex root as " + p); - } - return p; - } - - - public String getFilename() { return asRelativePath().toFile().getName(); } - - @Override - public void visitNodeUrl(Consumer fn) { - fn.accept(this); - } - - @Override - public Optional getNodeUrl() { - return Optional.of(this); - } - @Override - public int compareTo(@NotNull MemexNodeUrl o) { - return url.compareTo(o.getUrl()); - } - - public MemexNode toNode() { - return new MemexNode(this); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskState.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskState.java deleted file mode 100644 index 128bb918..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskState.java +++ /dev/null @@ -1,30 +0,0 @@ -package nu.marginalia.memex.memex.model; - -public enum MemexTaskState { - DONE('/', true,"done"), - SKIP('x', true,"skip"), - SKIP2('-', true,"skip"), - UNKNOWN('?', false, "unknown"), - URGENT('!', false, "urgent"), - TODO(0, false, "todo"); - - public final int key; - public final String style; - public final boolean done; - - MemexTaskState(int key, boolean done, String style) { - this.key = key; - this.style = style; - this.done = done; - } - - public static MemexTaskState of(MemexTaskTags tags) { - for (MemexTaskState state : values()) { - if (tags.hasTag(state.key)) { - return state; - } - } - return TODO; - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskTags.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskTags.java deleted file mode 100644 index b3906241..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexTaskTags.java +++ /dev/null @@ -1,40 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import lombok.Getter; - -import java.util.stream.Collectors; - -@Getter -public class MemexTaskTags { - public final String tagsCondensed; - - private static final int TAG_START = '('; - private static final int TAG_END = ')'; - - public MemexTaskTags(String text) { - tagsCondensed = getTags(text); - } - - public boolean hasTag(int tag) { - return tagsCondensed.indexOf(tag) >= 0; - } - - @Override - public String toString() { - return tagsCondensed.chars().mapToObj(c -> '(' + Character.toString(c) + ')') - .collect(Collectors.joining(" ")); - } - - private static String getTags(String task) { - StringBuilder sb = new StringBuilder(); - for (int i = task.indexOf(TAG_START); - i >= 0 && i+2 < task.length(); - i = task.indexOf(TAG_START, i+1)) - { - if (task.charAt(i+2) == TAG_END) { - sb.append(task.charAt(i+1)); - } - } - return sb.toString(); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexUrl.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexUrl.java deleted file mode 100644 index 9597a997..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/MemexUrl.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import java.util.Optional; -import java.util.function.Consumer; - -public interface MemexUrl { - String getUrl(); - - default void visitNodeUrl(Consumer fn) {} - default void visitExternalUrl(Consumer fn) {} - default Optional getNodeUrl() { return Optional.empty(); } - default Optional getExternUrl() { return Optional.empty(); } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexDirectory.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexDirectory.java deleted file mode 100644 index a198faf8..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexDirectory.java +++ /dev/null @@ -1,30 +0,0 @@ -package nu.marginalia.memex.memex.model.fs; - -import lombok.Getter; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.HashMap; -import java.util.Map; - -@Getter -public class MemexDirectory { - private final Map documents; - private final Map images; - private final Map subdirs; - - public MemexDirectory() { - documents = new HashMap<>(); - images = new HashMap<>(); - subdirs = new HashMap<>(); - } - - public void removeDocument(MemexNodeUrl url) { - documents.remove(url); - } - - public void removeImage(MemexNodeUrl url) { - images.remove(url); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexFileSystem.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexFileSystem.java deleted file mode 100644 index b5452f29..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/fs/MemexFileSystem.java +++ /dev/null @@ -1,88 +0,0 @@ -package nu.marginalia.memex.memex.model.fs; - -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -public class MemexFileSystem { - private final Map fileSystemContentsByDir = new ConcurrentHashMap<>(); - - public MemexFileSystem() { - } - - public Optional get(MemexNodeUrl url) { - return Optional.ofNullable(fileSystemContentsByDir.get(url)); - } - public List getDocuments(MemexNodeUrl url) { - var contents = fileSystemContentsByDir.get(url); - if (contents == null) { - return Collections.emptyList(); - } - var list = new ArrayList<>(contents.getDocuments().values()); - list.sort(Comparator.comparing(GemtextDocument::getUrl)); - return list; - } - - public List getImages(MemexNodeUrl url) { - var contents = fileSystemContentsByDir.get(url); - if (contents == null) { - return Collections.emptyList(); - } - var list = new ArrayList<>(contents.getImages().values()); - list.sort(Comparator.comparing(MemexImage::getPath)); - return list; - } - - public List getSubdirs(MemexNodeUrl url) { - var contents = fileSystemContentsByDir.get(url); - if (contents == null) { - return Collections.emptyList(); - } - var list = new ArrayList<>(contents.getSubdirs().keySet()); - list.sort(Comparator.naturalOrder()); - return list; - } - - public void recalculateDirectories() { - fileSystemContentsByDir.forEach((k, v) -> { - var parent = k.getParentUrl(); - if (parent != null) { - registerDir(k.getParentUrl()).getSubdirs().put(k, v); - } - }); - } - - public MemexDirectory registerDir(MemexNodeUrl url) { - return fileSystemContentsByDir - .computeIfAbsent(url, p -> new MemexDirectory()); - } - - public void register(MemexImage image) { - registerDir(image.path.getParentUrl()) - .getImages() - .put(image.path, image); - } - - public void register(GemtextDocument document) { - registerDir(document.getUrl().getParentUrl()) - .getDocuments() - .put(document.getUrl(), document); - } - - public void remove(MemexNodeUrl url) { - var contents = fileSystemContentsByDir.get(url.getParentUrl()); - contents.removeDocument(url); - contents.removeImage(url); - } - - public List getAllDirectories() { - return new ArrayList<>(fileSystemContentsByDir.keySet()); - } - - public boolean isDirectory(MemexNodeUrl url) { - return fileSystemContentsByDir.containsKey(url); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderCreateFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderCreateFormModel.java deleted file mode 100644 index c317ac82..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderCreateFormModel.java +++ /dev/null @@ -1,32 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -@RequiredArgsConstructor -@Getter -public class MemexRenderCreateFormModel implements MemexRendererableDirect { - public final MemexNodeUrl url; - public final List docs; - - public String getFilename() { - return url.getFilename(); - } - - public List getDocs() { - return docs.stream().sorted(Comparator.comparing(GemtextDocument::getUrl).reversed()).collect(Collectors.toList()); - } - - @Override - public String render(MemexHtmlRenderer renderer) { - return renderer.renderModel(this); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUpdateFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUpdateFormModel.java deleted file mode 100644 index 12ffefff..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUpdateFormModel.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -@AllArgsConstructor @Getter -public class MemexRenderUpdateFormModel implements MemexRendererableDirect { - public final MemexNodeUrl url; - public final String title; - public final String section; - public final String text; - - @Override - public String render(MemexHtmlRenderer renderer) { - return renderer.renderModel(this); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUploadFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUploadFormModel.java deleted file mode 100644 index 99cd3d36..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRenderUploadFormModel.java +++ /dev/null @@ -1,32 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - -@RequiredArgsConstructor -@Getter -public class MemexRenderUploadFormModel implements MemexRendererableDirect { - public final MemexNodeUrl url; - public final List docs; - - public String getFilename() { - return url.getFilename(); - } - - public List getDocs() { - return docs.stream().sorted(Comparator.comparing(GemtextDocument::getUrl).reversed()).collect(Collectors.toList()); - } - - @Override - public String render(MemexHtmlRenderer renderer) { - return renderer.renderModel(this); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererDeleteFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererDeleteFormModel.java deleted file mode 100644 index c0b71441..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererDeleteFormModel.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -@AllArgsConstructor @Getter -public class MemexRendererDeleteFormModel implements MemexRendererableDirect { - private final String doc; - private final MemexRendererImageModel image; - private final MemexNodeUrl url; - private final String type; - - @Override - public String render(MemexHtmlRenderer renderer) { - return renderer.renderModel(this); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererImageModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererImageModel.java deleted file mode 100644 index 79880110..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererImageModel.java +++ /dev/null @@ -1,36 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.SneakyThrows; -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.nio.file.Files; -import java.util.Base64; -import java.util.List; - -@AllArgsConstructor @Getter -public class MemexRendererImageModel { - public final MemexImage image; - public final List backlinks; - - public final String parent; - - public String getParent() { - if ("/".equals(parent) || parent.isBlank()) { - return null; - } - return parent; - } - - public MemexNodeUrl getPath() { - return image.path; - } - - @SneakyThrows - public String getData() { - return Base64.getEncoder().encodeToString(Files.readAllBytes(image.realPath)); - } -} \ No newline at end of file diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererIndexModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererIndexModel.java deleted file mode 100644 index 13630727..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererIndexModel.java +++ /dev/null @@ -1,71 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.model.MemexImage; -import nu.marginalia.memex.memex.model.MemexIndexTask; -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -@RequiredArgsConstructor -@Getter -public class MemexRendererIndexModel { - public final MemexNodeUrl url; - public final List docs; - public final List images; - public final List directories; - public final List tasks; - public final List backlinks; - - public String getFilename() { - return url.getFilename(); - } - - public MemexNodeUrl getParent() { - return url.getParentUrl(); - } - - public List getDocs() { - return docs.stream() - .filter(doc -> !doc.isIndex()) - .sorted(Comparator.comparing(GemtextDocument::getUrl).reversed()) - .collect(Collectors.toList()); - } - - public final String getTitle() { - return Optional.ofNullable(getIndexDocument()).map(GemtextDocument::getTitle).orElse(url.toString()); - } - - public GemtextDocument getDocument(String filename) { - return docs.stream().filter(doc -> doc.getUrl().getFilename().endsWith(filename)).findFirst().orElse(null); - } - - private GemtextDocument getIndexDocument() { - return getDocument("index.gmi"); - } - - public String getIndexData() { - var indexDoc = getIndexDocument(); - if (indexDoc == null) { - return null; - } - var htmlRenderer = new GemtextRendererFactory("").htmlRendererReadOnly(); - return indexDoc.render(htmlRenderer); - } - - public boolean hasPragma(String value) { - var doc = getIndexDocument(); - if (doc == null) { - return false; - } - return doc.getPragmas().contains(value); - - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererRenameFormModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererRenameFormModel.java deleted file mode 100644 index 464bca88..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererRenameFormModel.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -@AllArgsConstructor @Getter -public class MemexRendererRenameFormModel implements MemexRendererableDirect { - private final String doc; - private final MemexRendererImageModel image; - private final MemexNodeUrl url; - private final String type; - - @Override - public String render(MemexHtmlRenderer renderer) { - return renderer.renderModel(this); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererTombstoneModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererTombstoneModel.java deleted file mode 100644 index aebcb9b1..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererTombstoneModel.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -import java.util.List; - -@AllArgsConstructor @Getter -public class MemexRendererTombstoneModel { - private final MemexNodeUrl url; - private final String message; - private final String redirect; - public final List backlinks; -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererViewModel.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererViewModel.java deleted file mode 100644 index 8aac05e8..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererViewModel.java +++ /dev/null @@ -1,24 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexLink; - -import java.util.List; - -@AllArgsConstructor @Getter -public class MemexRendererViewModel { - public final GemtextDocument baseDoc; - public final String title; - public final List backlinks; - public final String doc; - public final String parent; - - public String getParent() { - if ("/".equals(parent) || parent.isBlank()) { - return null; - } - return parent; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererableDirect.java b/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererableDirect.java deleted file mode 100644 index b0631dfa..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/model/render/MemexRendererableDirect.java +++ /dev/null @@ -1,7 +0,0 @@ -package nu.marginalia.memex.memex.model.render; - -import nu.marginalia.memex.memex.renderer.MemexHtmlRenderer; - -public interface MemexRendererableDirect { - String render(MemexHtmlRenderer renderer); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexGmiRenderer.java b/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexGmiRenderer.java deleted file mode 100644 index c74b498b..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexGmiRenderer.java +++ /dev/null @@ -1,228 +0,0 @@ -package nu.marginalia.memex.memex.renderer; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.MemexData; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Comparator; - -public class MemexGmiRenderer { - private final MemexFileWriter renderedResources; - private final MemexData data; - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Inject - public MemexGmiRenderer(@Named("gmi") MemexFileWriter renderedResources, - MemexData data) - { - this.renderedResources = renderedResources; - this.data = data; - } - - - public void render(MemexNodeUrl url) { - if (data.getDocument(url) != null) { - renderDocument(url); - } - else if(data.getImage(url) != null) { - renderImage(url); - } - else if(data.isDirectory(url)) { - renderIndex(url); - } - else if(data.hasTombstone(url)) { - renderTombstone(url); - } - else { - logger.warn("I don't know how to render {}", url); - } - } - - private void renderDocument(MemexNodeUrl url) { - if ("index.gmi".equals(url.getFilename())) { - return; - } - - var doc = data.getDocument(url); - var renderer = new GemtextRendererFactory().gemtextRendererPublic(); - - try { - renderedResources.write(url, (w) -> { - doc.render(renderer, w); - w.println(); - backlinks(w, url); - w.println("# Navigation\n"); - w.printf("=> %s Back to Index\n", url.getParentUrl()); - w.println("\nReach me at kontakt@marginalia.nu"); - }); - } catch (IOException e) { - logger.error("Failed to render document " + url, e); - } - - } - - private void renderImage(MemexNodeUrl url) { - try { - renderedResources.write(url, data.getImage(url).realPath); - } catch (IOException e) { - logger.error("Failed to image document " + url, e); - } - } - - private void renderIndex(MemexNodeUrl url) { - - var renderer = new GemtextRendererFactory().gemtextRendererPublic(); - - var doc = data.getDocument(url.child("index.gmi")); - boolean feed = doc != null && doc.getPragmas().contains("FEED"); - boolean listing = doc != null && doc.getPragmas().contains("LISTING"); - - try { - renderedResources.write(url.child("index.gmi"), (w) -> { - - if (null != doc) doc.render(renderer, w); - else w.printf("# %s\n", url); - - - if (listing) { - documentsInUrlListing(url, w); - } - - if (feed) { - w.printf("\n=> %s/feed.gmi Clean gemsub feed\n", url); - w.printf("=> %s/feed.xml Atom feed\n", url); - } - - w.println("\n# Directory Contents\n"); - directoriesInUrl(url, w); - if (!listing) { - documentsInUrl(url, w); - } - imagesInUrl(url, w); - backlinks(w, url, url.child("index.gmi")); - w.println("\nReach me at kontakt@marginalia.nu"); - - }); - - if (feed) { - renderedResources.write(url.child("feed.gmi"), (w) -> { - w.printf("# marginalia.nu%s\n", url); - w.println(); - var docs = data.getDocumentsByPath(url); - docs.sort(Comparator.comparing(GemtextDocument::getUrl).reversed()); - for (var d : docs) { - if (d.getUrl().getFilename().equals("index.gmi")) { - continue; - } - if (d.getPragmas().contains("DRAFT")) { - continue; - } - w.printf("=> gemini://marginalia.nu%s\t%s %s\n", d.getUrl(), d.getDate(), d.getTitle().replaceAll("\\[[^\\]]+\\]", "")); - } - }); - } - } catch (IOException e) { - logger.error("Failed to render document " + url, e); - } - } - - private void backlinks(PrintWriter w, MemexNodeUrl... urls) { - var bls = data.getBacklinks(urls); - if (!bls.isEmpty()) { - w.println("\n# Backlinks\n"); - for (var bl : bls) { - w.printf("=> %s\n", bl.src); - } - w.println(); - } - } - - private void documentsInUrl(MemexNodeUrl url, PrintWriter w) { - var docs = data.getDocumentsByPath(url); - if (docs.size() > (data.getDocument(url.child("index.gmi")) == null ? 0 : 1)) { - for (var d : docs) { - if (d.getUrl().getFilename().equals("index.gmi")) { - continue; - } - w.printf("=> %s\t\uD83D\uDDD2 ️️️%s\n", d.getUrl(), d.getTitle()); - } - w.println(); - } - } - - private void documentsInUrlListing(MemexNodeUrl url, PrintWriter w) { - var docs = data.getDocumentsByPath(url); - - docs.sort(Comparator.comparing(GemtextDocument::getUrl).reversed()); - - if (!docs.isEmpty()) { - for (var d : docs) { - if (d.getUrl().getFilename().equals("index.gmi")) { - continue; - } - w.printf("=> %s\t%s\n", d.getUrl(), d.getTitle()); - } - w.println(); - } - } - - private void imagesInUrl(MemexNodeUrl url, PrintWriter w) { - var images = data.getImagesByPath(url); - if (!images.isEmpty()) { - for (var i : images) { - w.printf("=> %s \uD83D\uDDBC️ %s\n", i.path, i.path.getFilename()); - } - w.println(); - } - } - - private void directoriesInUrl(MemexNodeUrl url, PrintWriter w) { - var dirs = data.getSubdirsByPath(url); - final boolean isRoot = url.getParentUrl() == null; - if (isRoot && dirs.isEmpty()) { - return; - } - - if (!isRoot) { - w.println("=> ../ ⬆ ../ "); - - } - if (dirs.isEmpty()) { - w.println(); - } - for (var d : dirs) { - w.printf("=> %s \uD83D\uDDC2️ %s/\n", d, d.getFilename()); - } - } - - - private void renderTombstone(MemexNodeUrl url) { - String message = data.getTombstones().flatMap(tombstones -> tombstones.getLinkData(url)).orElse(null); - String redir = data.getRedirects().flatMap(redirects -> redirects.getLinkData(url)).orElse(null); - - try { - renderedResources.write(url, w -> { - w.printf("# %s is gone\n\n", url); - if (message != null) { - w.printf("%s\n", message); - } - if (redir != null) { - w.println("Please see"); - w.printf("=> %s\n", redir); - } - backlinks(w, url); - w.println("\nReach me at kontakt@marginalia.nu"); - }); - } catch (IOException e) { - logger.error("Failed to render tombstone " + url, e); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexHtmlRenderer.java b/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexHtmlRenderer.java deleted file mode 100644 index 6cfaada1..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexHtmlRenderer.java +++ /dev/null @@ -1,199 +0,0 @@ -package nu.marginalia.memex.memex.renderer; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.gmi.renderer.GemtextRendererFactory; -import nu.marginalia.memex.memex.MemexData; -import nu.marginalia.memex.memex.model.MemexIndexTask; -import nu.marginalia.memex.memex.model.MemexLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.render.*; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.renderer.MustacheRenderer; -import nu.marginalia.memex.renderer.RendererFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoField; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -public class MemexHtmlRenderer { - - private final MemexFileWriter htmlRenderedResources; - private final MemexFileWriter gmiRenderedResources; - - private final MemexData data; - - private final MustacheRenderer viewRenderer; - private final MustacheRenderer indexRenderer; - private final MustacheRenderer indexFeedRenderer; - private final MustacheRenderer imageRenderer; - private final MustacheRenderer tombstoneRenderer; - - private final MustacheRenderer updateFormRenderer; - private final MustacheRenderer uploadFormRenderer; - private final MustacheRenderer createFormRenderer; - private final MustacheRenderer deleteFormRenderer; - private final MustacheRenderer renameFormRenderer; - - private final Logger logger = LoggerFactory.getLogger(getClass()); - - @Inject - public MemexHtmlRenderer( - @Named("html") MemexFileWriter htmlRenderedResources, - @Named("gmi") MemexFileWriter gmiRenderedResources, - MemexData data) throws IOException { - this.htmlRenderedResources = htmlRenderedResources; - this.gmiRenderedResources = gmiRenderedResources; - this.data = data; - - final var rendererFactory = new RendererFactory(); - - viewRenderer = rendererFactory.renderer("static/memex/memex-view"); - indexRenderer = rendererFactory.renderer("static/memex/memex-index"); - indexFeedRenderer = rendererFactory.renderer("static/memex/memex-index-feed"); - imageRenderer = rendererFactory.renderer("static/memex/memex-image"); - - tombstoneRenderer = rendererFactory.renderer("static/memex/memex-tombstone"); - - updateFormRenderer = rendererFactory.renderer("static/memex/memex-update-form"); - uploadFormRenderer = rendererFactory.renderer("static/memex/memex-upload-form"); - deleteFormRenderer = rendererFactory.renderer("static/memex/memex-delete-form"); - renameFormRenderer = rendererFactory.renderer("static/memex/memex-rename-form"); - createFormRenderer = rendererFactory.renderer("static/memex/memex-create-form"); - - } - - public void render(MemexNodeUrl url) { - if (data.getDocument(url) != null) { - renderDocument(url); - } - else if(data.getImage(url) != null) { - renderImage(url); - } - else if(data.isDirectory(url)) { - renderIndex(url); - } - else if(data.hasTombstone(url)) { - renderTombstone(url); - } - else { - logger.warn("I don't know how to render {}", url); - } - } - - public void renderDocument(MemexNodeUrl url) { - var doc = Objects.requireNonNull(data.getDocument(url), "could not get document " + url); - var htmlRenderer = new GemtextRendererFactory("", url.toString()).htmlRendererEditable(); - var model = new MemexRendererViewModel(doc, - doc.getTitle(), - data.getBacklinks(url), - doc.render(htmlRenderer), - url.getParentStr() - ); - - try { - htmlRenderedResources.write(url, viewRenderer.render(model, Map.of("urlRoot", ""))); - } catch (IOException e) { - logger.error("Failed to render document " + url, e); - } - - } - - public void renderIndex(MemexNodeUrl url) { - - var docs = data.getDocumentsByPath(url); - var images = data.getImagesByPath(url); - var dirs = data.getSubdirsByPath(url); - - var tasks = docs.stream().flatMap(doc -> doc.getOpenTopTasks().entrySet() - .stream() - .sorted(Map.Entry.comparingByKey()) - .map(entry -> new MemexIndexTask(entry.getValue().getLeft(), - entry.getKey().toString(), - doc.getUrl().toString(), - entry.getValue().getRight().style)) - ).collect(Collectors.toList()); - - List backlinks = data.getBacklinks(url, url.child("index.gmi")); - var model = new MemexRendererIndexModel(url, docs, images, new ArrayList<>(dirs), tasks, backlinks); - - try { - htmlRenderedResources.write(url.child("index.html"), indexRenderer.render(model, Map.of("urlRoot", ""))); - if (model.hasPragma("FEED")) { - String nowStr = OffsetDateTime.now().with(ChronoField.MILLI_OF_SECOND, 0).format(DateTimeFormatter.ISO_DATE_TIME); - htmlRenderedResources.write(url.child("feed.xml"), indexFeedRenderer.render(model, - Map.of("domain", "https://memex.marginalia.nu", "now", nowStr))); - gmiRenderedResources.write(url.child("feed.xml"), indexFeedRenderer.render(model, - Map.of("domain", "gemini://marginalia.nu", "now", nowStr))); - } - } catch (IOException e) { - logger.error("Failed to render index model " + url, e); - } - } - - public void renderImage(MemexNodeUrl url) { - var img = data.getImage(url); - var backlinks = data.getBacklinks(img.path); - var parent = img.path.getParentStr(); - var model = new MemexRendererImageModel(img, backlinks, parent); - - try { - htmlRenderedResources.write(img.path, imageRenderer.render(model, Map.of("urlRoot", ""))); - } catch (IOException e) { - logger.error("Failed to render image model " + img.path, e); - } - } - - public void renderTombstone(MemexNodeUrl url) { - - String message = data.getTombstones().flatMap(tombstones -> tombstones.getLinkData(url)).orElse(null); - String redir = data.getRedirects().flatMap(redirects -> redirects.getLinkData(url)).orElse(null); - - var model = new MemexRendererTombstoneModel(url, - message, - redir, - data.getBacklinks(url)); - - try { - htmlRenderedResources.write(url, tombstoneRenderer.render(model, Map.of("urlRoot", ""))); - } catch (IOException e) { - logger.error("Failed to render tombstone model " + url, e); - } - } - - @SneakyThrows - public String renderModel(MemexRendererDeleteFormModel model) { - return deleteFormRenderer.render(model, Map.of("urlRoot", "")); - } - - - @SneakyThrows - public String renderModel(MemexRendererRenameFormModel model) { - return renameFormRenderer.render(model, Map.of("urlRoot", "")); - } - - @SneakyThrows - public String renderModel(MemexRenderCreateFormModel model) { - return createFormRenderer.render(model, Map.of("urlRoot", "")); - } - - @SneakyThrows - public String renderModel(MemexRenderUploadFormModel model) { - return uploadFormRenderer.render(model, Map.of("urlRoot", "")); - } - - @SneakyThrows - public String renderModel(MemexRenderUpdateFormModel model) { - return updateFormRenderer.render(model, Map.of("urlRoot", "")); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexRendererers.java b/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexRendererers.java deleted file mode 100644 index bffaaf26..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/renderer/MemexRendererers.java +++ /dev/null @@ -1,22 +0,0 @@ -package nu.marginalia.memex.memex.renderer; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -@Singleton -public class MemexRendererers { - private final MemexGmiRenderer gmiRenderer; - private final MemexHtmlRenderer htmlRenderer; - - @Inject - public MemexRendererers(MemexGmiRenderer gmiRenderer, MemexHtmlRenderer htmlRenderer) { - this.gmiRenderer = gmiRenderer; - this.htmlRenderer = htmlRenderer; - } - - public void render(MemexNodeUrl url) { - gmiRenderer.render(url); - htmlRenderer.render(url); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemModifiedTimes.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemModifiedTimes.java deleted file mode 100644 index ceb5f38f..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemModifiedTimes.java +++ /dev/null @@ -1,23 +0,0 @@ -package nu.marginalia.memex.memex.system; - -import com.google.inject.Singleton; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; - -@Singleton -public class MemexFileSystemModifiedTimes { - - private final Map modifiedTimes = new ConcurrentHashMap<>(); - - public boolean isFreshUpdate(Path node) throws IOException { - long mtime = Files.getLastModifiedTime(node).toMillis(); - - return !Objects.equals(modifiedTimes.put(node, mtime), mtime); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemMonitor.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemMonitor.java deleted file mode 100644 index e5c1a94a..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileSystemMonitor.java +++ /dev/null @@ -1,116 +0,0 @@ -package nu.marginalia.memex.memex.system; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import lombok.SneakyThrows; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.*; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import static java.nio.file.StandardWatchEventKinds.*; - -public class MemexFileSystemMonitor { - private final WatchService watchService; - private final Set updatedUrls = new HashSet<>(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - - private final Map roots = new ConcurrentHashMap<>(); - private final Path memexRoot; - - @Inject - public MemexFileSystemMonitor(@Named("memex-root") Path monitorPath) throws IOException { - this.memexRoot = monitorPath; - this.watchService = FileSystems.getDefault().newWatchService(); - - registerWatcher(monitorPath); - - try (var files = Files.walk(monitorPath)) { - files.filter(Files::isDirectory).forEach(this::registerWatcher); - } - - var monitorThread = new Thread(this::monitorWatch, getClass().getSimpleName()); - monitorThread.setDaemon(true); - monitorThread.start(); - } - - private void registerWatcher(Path path) { - if (path.toString().contains(".git")) { - return; - } - - try { - logger.info("Watching " + path); - var key = path.register(watchService, - StandardWatchEventKinds.ENTRY_CREATE, - StandardWatchEventKinds.ENTRY_MODIFY, - StandardWatchEventKinds.ENTRY_DELETE); - roots.put(key, path); - - } catch (IOException e) { - logger.error("Failed to register directory watcher on " + path, e); - } - } - - public List getUpdatedUrls() { - synchronized (updatedUrls) { - if (updatedUrls.isEmpty()) { - return Collections.emptyList(); - } - var ret = new ArrayList<>(updatedUrls); - updatedUrls.clear(); - return ret; - } - } - - - @SneakyThrows - @SuppressWarnings("unchecked") - private void monitorWatch() { - for (;;) { - var key = watchService.take(); - - for (var evt : key.pollEvents()) { - var kind = evt.kind(); - - if (kind == OVERFLOW) { - var root = roots.get(key); - - try (var files = Files.list(root)) { - files.forEach(file -> - updatedUrls.add(MemexNodeUrl.ofRelativePath(memexRoot, file)) - ); - } - - continue; - } - - WatchEvent ev = (WatchEvent)evt; - Path root = roots.get(key); - Path filename = ev.context(); - Path absPath = root.resolve(filename); - - - if (kind == ENTRY_CREATE && Files.isDirectory(absPath)) { - registerWatcher(absPath); - } - - MemexNodeUrl url = MemexNodeUrl.ofRelativePath(memexRoot, absPath); - synchronized (updatedUrls) { - updatedUrls.add(url); - } - - } - - boolean valid = key.reset(); - if (!valid) { - logger.info("Deregistering key for " + roots.get(key)); - roots.remove(key); - } - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileWriter.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileWriter.java deleted file mode 100644 index 285d149e..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexFileWriter.java +++ /dev/null @@ -1,120 +0,0 @@ -package nu.marginalia.memex.memex.system; - -import nu.marginalia.util.FileSizeUtil; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Set; - -public class MemexFileWriter { - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Path renderedResourcesRoot; - - - private final Set filePermission = Set.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.OTHERS_READ); - - private final Set dirPermission = Set.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, - PosixFilePermission.OWNER_EXECUTE, - PosixFilePermission.GROUP_READ, - PosixFilePermission.GROUP_EXECUTE, - PosixFilePermission.OTHERS_READ, - PosixFilePermission.OTHERS_EXECUTE); - - public MemexFileWriter(Path renderedResourcesRoot) { - this.renderedResourcesRoot = renderedResourcesRoot; - } - - public boolean exists(MemexNodeUrl url) { - return Files.exists(getPath(url)); - } - - public void write(MemexNodeUrl url, String contents) throws IOException { - logger.info("write({},{})", url, FileSizeUtil.readableSize(contents.length())); - var destPath = getPath(url); - var tempFile = Files.createTempFile(renderedResourcesRoot, url.getFilename(), ".tmp"); - ensureDirectoryExists(destPath.getParent()); - - Files.createDirectories(destPath.getParent()); - Files.writeString(tempFile, contents); - Files.move(tempFile, destPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - setFilePermissions(destPath); - } - - private void ensureDirectoryExists(Path dir) throws IOException { - Files.createDirectories(dir); - setDirPermissions(dir); - } - - private void setDirPermissions(Path dir) throws IOException { - for (var rel = renderedResourcesRoot.relativize(dir); rel != null; rel = rel.getParent()) { - Files.setPosixFilePermissions(renderedResourcesRoot.resolve(rel), dirPermission); - } - } - - public void write(MemexNodeUrl url, byte[] contents) throws IOException { - logger.info("write({}, {})", url, FileSizeUtil.readableSize(contents.length)); - - var destPath = getPath(url); - var tempFile = Files.createTempFile(renderedResourcesRoot, url.getFilename(), ".tmp"); - ensureDirectoryExists(destPath.getParent()); - Files.write(tempFile, contents); - Files.move(tempFile, destPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - setFilePermissions(destPath); - } - - public void write(MemexNodeUrl url, Path realPath) throws IOException { - logger.info("copy({} from {})", url, realPath); - - var destPath = getPath(url); - var tempFile = Files.createTempFile(renderedResourcesRoot, url.getFilename(), ".tmp"); - ensureDirectoryExists(destPath.getParent()); - Files.copy(realPath, tempFile, StandardCopyOption.REPLACE_EXISTING); - Files.move(tempFile, destPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - setFilePermissions(destPath); - } - - public void write(MemexNodeUrl url, WriteOperation wo) throws IOException { - logger.info("write({}, streamed)", url); - - var destPath = getPath(url); - var tempFile = Files.createTempFile(renderedResourcesRoot, url.getFilename(), ".tmp"); - ensureDirectoryExists(destPath.getParent()); - - try (var os = new PrintWriter(Files.newOutputStream(tempFile))) { - wo.write(os); - } - - Files.move(tempFile, destPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - setFilePermissions(destPath); - } - - - private void setFilePermissions(Path destPath) throws IOException { - Files.setPosixFilePermissions(destPath, filePermission); - } - - private Path getPath(MemexNodeUrl url) { - final Path path = Path.of(renderedResourcesRoot + url.toString()).normalize(); - - if (!path.startsWith(renderedResourcesRoot)) { - throw new IllegalStateException("URL " + url + " resulted in a path outside of root " + renderedResourcesRoot); - } - - return path; - } - - public interface WriteOperation { - void write(PrintWriter w) throws IOException; - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexSourceFileSystem.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexSourceFileSystem.java deleted file mode 100644 index 305a4c42..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/MemexSourceFileSystem.java +++ /dev/null @@ -1,87 +0,0 @@ -package nu.marginalia.memex.memex.system; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import nu.marginalia.memex.memex.system.git.MemexGitRepo; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.StandardOpenOption; - -@Singleton -public class MemexSourceFileSystem { - - private final Path root; - private final MemexGitRepo gitRepo; - - private static final Logger logger = LoggerFactory.getLogger(MemexSourceFileSystem.class); - - @Inject - public MemexSourceFileSystem(@Named("memex-root") Path root, - MemexGitRepo gitRepo) { - this.root = root; - this.gitRepo = gitRepo; - } - - public void pullChanges() { - gitRepo.pull(); - } - - public void replaceFile(MemexNodeUrl url, String text) throws IOException { - var path = url.asAbsolutePath(root); - Files.writeString(path, text, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - - gitRepo.update(url); - } - - public void createFile(MemexNodeUrl url, String text) throws IOException { - var path = url.asAbsolutePath(root); - logger.info("Writing {} ({}b)", path, text.length()); - - Files.writeString(path, text, StandardOpenOption.CREATE_NEW); - - gitRepo.add(url); - } - - public void createFile(MemexNodeUrl url, byte[] bytes) throws IOException { - var path = url.asAbsolutePath(root); - logger.info("Writing {} ({}b)", path, bytes.length); - - Files.write(path, bytes, StandardOpenOption.CREATE_NEW); - - gitRepo.add(url); - } - - public void delete(MemexNodeUrl url) throws IOException { - var path = url.asAbsolutePath(root); - - logger.info("Delete {}", path); - Files.delete(path); - - gitRepo.remove(url); - } - - public void renameFile(MemexNodeUrl src, MemexNodeUrl dst) throws IOException { - var srcPath = src.asAbsolutePath(root); - var dstPath = dst.asAbsolutePath(root); - - if (!Files.exists(srcPath) || Files.exists(dstPath)) { - throw new IOException("Could not rename " + src + " into " + dst); - } - - Files.move(srcPath, dstPath, StandardCopyOption.ATOMIC_MOVE); - gitRepo.rename(src, dst); - } - - public byte[] getRaw(MemexNodeUrl url) throws IOException { - logger.info("Getting raw file contents of {}", url); - - return Files.readAllBytes(url.asAbsolutePath(root)); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepo.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepo.java deleted file mode 100644 index a26375c9..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepo.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.memex.memex.system.git; - -import nu.marginalia.memex.memex.model.MemexNodeUrl; - -public interface MemexGitRepo { - void pull(); - - void remove(MemexNodeUrl url); - - void add(MemexNodeUrl url); - - void update(MemexNodeUrl url); - - void rename(MemexNodeUrl src, MemexNodeUrl dst); -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoDummy.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoDummy.java deleted file mode 100644 index 84cc3406..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoDummy.java +++ /dev/null @@ -1,36 +0,0 @@ -package nu.marginalia.memex.memex.system.git; - -import com.google.inject.Singleton; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Singleton -public class MemexGitRepoDummy implements MemexGitRepo { - private static final Logger logger = LoggerFactory.getLogger(MemexGitRepoDummy.class); - - @Override - public void pull() { - logger.info("Would perform a pull here"); - } - - @Override - public void remove(MemexNodeUrl url) { - logger.info("Would perform a remove here"); - } - - @Override - public void add(MemexNodeUrl url) { - logger.info("Would perform an add here"); - } - - @Override - public void update(MemexNodeUrl url) { - logger.info("Would perform an update here"); - } - - @Override - public void rename(MemexNodeUrl src, MemexNodeUrl dst) { - logger.info("Would perform a rename here"); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoImpl.java b/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoImpl.java deleted file mode 100644 index e60a9db0..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/memex/system/git/MemexGitRepoImpl.java +++ /dev/null @@ -1,141 +0,0 @@ -package nu.marginalia.memex.memex.system.git; - -import com.google.inject.Inject; -import com.google.inject.Singleton; -import com.google.inject.name.Named; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.JSchException; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.transport.JschConfigSessionFactory; -import org.eclipse.jgit.transport.SshSessionFactory; -import org.eclipse.jgit.util.FS; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Path; - -@Singleton -public class MemexGitRepoImpl implements MemexGitRepo { - - private final Git git; - private final Logger logger = LoggerFactory.getLogger(MemexGitRepoImpl.class); - - @Inject - public MemexGitRepoImpl(@Named("memex-root") Path root) throws IOException { - - FileRepositoryBuilder repositoryBuilder = new FileRepositoryBuilder(); - - SshSessionFactory.setInstance(new JschConfigSessionFactory() { - @Override - protected JSch createDefaultJSch(FS fs) throws JSchException { - JSch defaultJSch = super.createDefaultJSch(fs); - defaultJSch.addIdentity("/var/lib/wmsa/.ssh/id_rsa"); - return defaultJSch; - } - }); - - Repository repository = repositoryBuilder.setGitDir(root.resolve(".git").toFile()) - .readEnvironment() - .findGitDir() - .setMustExist(true) - .build(); - - git = new Git(repository); - - pull(); - } - - @Override - public void pull() { - try { - git.pull().call(); - } - catch (GitAPIException ex) { - logger.error("Git operation failed", ex); - } - } - - @Override - public void remove(MemexNodeUrl url) { - try { - git.rm() - .addFilepattern(filePattern(url)) - .call(); - - commit("Removing " + url); - push(); - } - catch (GitAPIException ex) { - logger.error("Git operation failed", ex); - } - } - - @Override - public void add(MemexNodeUrl url) { - try { - git.add() - .addFilepattern(filePattern(url)) - .call(); - - commit("Adding " + url); - push(); - - - } - catch (GitAPIException ex) { - logger.error("Git operation failed", ex); - } - } - @Override - public void update(MemexNodeUrl url) { - try { - git.add() - .setUpdate(true) - .addFilepattern(filePattern(url)) - .call(); - - commit("Update " + url); - push(); - - - } - catch (GitAPIException ex) { - logger.error("Git operation failed", ex); - } - } - - - @Override - public void rename(MemexNodeUrl src, MemexNodeUrl dst) { - try { - git.rm().addFilepattern(filePattern(src)).call(); - git.add().addFilepattern(filePattern(dst)).call(); - commit("Renaming " + src + " into " + dst); - push(); - } - catch (GitAPIException ex) { - logger.error("Git operation failed", ex); - } - } - - private void push() throws GitAPIException { - git.push().call(); - } - - private void commit(String message) throws GitAPIException { - git.commit() - .setCommitter("marginalia", "system@marginalia.nu") - .setMessage("Changes from web gui: " + message) - .call(); - } - - private String filePattern(MemexNodeUrl url) { - return url.asRelativePath().toString().replaceAll("^/+", ""); - } - -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/renderer/MustacheRenderer.java b/other/memex/src/main/java/nu/marginalia/memex/renderer/MustacheRenderer.java deleted file mode 100644 index 1c6e7e92..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/renderer/MustacheRenderer.java +++ /dev/null @@ -1,164 +0,0 @@ -package nu.marginalia.memex.renderer; - -import com.github.jknack.handlebars.*; -import com.github.jknack.handlebars.helper.ConditionalHelpers; -import com.github.jknack.handlebars.io.ClassPathTemplateLoader; -import com.github.jknack.handlebars.io.TemplateLoader; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.render.MemexRendererIndexModel; -import nu.marginalia.memex.memex.model.render.MemexRendererViewModel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.file.Path; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class MustacheRenderer { - Template template; - private final Logger logger = LoggerFactory.getLogger(getClass()); - - MustacheRenderer(String templateFile) throws IOException { - - TemplateLoader loader = new ClassPathTemplateLoader(); - loader.setPrefix("/templates"); - loader.setSuffix(".hdb"); - - var handlebars = new Handlebars(loader); - handlebars.registerHelpers(ConditionalHelpers.class); - handlebars.registerHelpers(new GeminiHelpers()); - handlebars.registerHelper("md", new MarkdownHelper()); - handlebars.registerHelper("gen-url", this::genereraUrl); - handlebars.registerHelper("gen-thread-url", this::genereraTradUrl); - handlebars.registerHelper("gen-author-url", this::generereraAuthorUrl); - - - - try { - template = handlebars.compile(templateFile); - } - catch (FileNotFoundException ex) { - logger.error("Kunde inte ladda template " + templateFile, ex); - System.exit(2); - } - catch (HandlebarsException ex) { - logger.error("Kunde inte instantiera mall " + templateFile, ex); - System.exit(2); - } - } - - public static final class GeminiHelpers { - - public CharSequence pragma(Options options) throws IOException { - var model = options.context.model(); - GemtextDocument doc; - if (model instanceof MemexRendererIndexModel) { - doc = ((MemexRendererIndexModel) model).getDocument("index.gmi"); - } - else if (model instanceof MemexRendererViewModel) { - doc = ((MemexRendererViewModel)model).baseDoc; - } - else { - doc = null; - } - - if (doc != null && doc.getPragmas().contains((String) options.param(0))) { - return options.fn(options.context); - } - return null; - } - public CharSequence amgarp(Options options) throws IOException { - var model = options.context.model(); - GemtextDocument doc; - if (model instanceof MemexRendererIndexModel) { - doc = ((MemexRendererIndexModel) model).getDocument("index.gmi"); - } - else if (model instanceof MemexRendererViewModel) { - doc = ((MemexRendererViewModel)model).baseDoc; - } - else { - doc = null; - } - - if (doc == null || !doc.getPragmas().contains((String) options.param(0))) { - return options.fn(options.context); - } - return null; - } - - public CharSequence topbar(MemexNodeUrl url, Options options) throws IOException { - var path = url.asRelativePath(); - LinkedList nodes = new LinkedList<>(); - - for (Path p = path; p != null; p = p.getParent()) { - nodes.addFirst(p); - } - StringBuilder sb = new StringBuilder(); - for (var p : nodes) { - String name = p.toFile().getName(); - String type = "dir"; - if ("".equals(name)) { - name = "marginalia"; - type = "root"; - } - if (p.equals(path) && name.contains(".")) { - type = "file"; - } - Context newCtx = Context.newBlockParamContext(options.context, - List.of("url", "name", "type"), - List.of(p, name, type) - ); - sb.append(options.fn(newCtx)); - } - return sb.toString(); - } - } - - @SneakyThrows - public String render(T model) { - return template.apply(model); - } - - @SneakyThrows - public String render(T model, String name, List children) { - Context ctx = Context.newBuilder(model).combine(name, children).build(); - - return template.apply(ctx); - } - - @SneakyThrows - public String render(T model, Map children) { - Context ctx = Context.newBuilder(model).combine(children).build(); - return template.apply(ctx); - } - - private Object genereraUrl(Object context, Options options) { - if (null != context) { - return context.toString().toLowerCase() + ".html"; - } else { - logger.error("Kunde inte generera URL, blockParams {}", options.blockParams); - return ""; - } - } - private Object genereraTradUrl(Object context, Options options) { - if (null != context) { - return context.toString().toLowerCase() + "/view.html"; - } else { - logger.error("Kunde inte generera URL, blockParams {}", options.blockParams); - return ""; - } - } - private Object generereraAuthorUrl(Object context, Options options) { - if (null != context) { - return "u_" + context.toString().toLowerCase() + ".html"; - } else { - logger.error("Kunde inte generera URL, blockParams {}", options.blockParams); - return ""; - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/renderer/RendererFactory.java b/other/memex/src/main/java/nu/marginalia/memex/renderer/RendererFactory.java deleted file mode 100644 index 96e61038..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/renderer/RendererFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.memex.renderer; - -import java.io.IOException; - -public class RendererFactory { - - public RendererFactory() { - } - - public MustacheRenderer renderer(String template) throws IOException { - return new MustacheRenderer<>(template); - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/util/dithering/FloydSteinbergDither.java b/other/memex/src/main/java/nu/marginalia/memex/util/dithering/FloydSteinbergDither.java deleted file mode 100644 index 942d65b0..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/util/dithering/FloydSteinbergDither.java +++ /dev/null @@ -1,243 +0,0 @@ -package nu.marginalia.memex.util.dithering; - -import lombok.AllArgsConstructor; -import net.sf.image4j.util.ConvertUtil; -import org.imgscalr.Scalr; - -import java.awt.image.BufferedImage; -import java.awt.image.IndexColorModel; -import java.util.Arrays; -import java.util.Comparator; - -public class FloydSteinbergDither { - private final Color[] palette; - - private final int maxWidth; - private final int maxHeight; - - public FloydSteinbergDither(int[] colors, int maxWidth, int maxHeight) { - this.maxWidth = maxWidth; - this.maxHeight = maxHeight; - - palette = Arrays.stream(colors) - .mapToObj(Color::new) - .toArray(Color[]::new); - } - - public BufferedImage convert(BufferedImage src) { - BufferedImage out = dither(resize(src)); - - if (palette.length <= 16) { - int[] cmap = new int[palette.length]; - for (int i = 0; i < palette.length; i++) { - cmap[i] = palette[i].toInt(); - } - return ConvertUtil.convert4(out, cmap); - } - return out; - } - - private BufferedImage dither(BufferedImage in) { - - Errors errors = new Errors(in.getWidth(), in.getHeight()); - - final BufferedImage out = createOutBuffer(in); - - for (int y = 0; y < in.getHeight(); y++) { - for (int x = 0; x < in.getWidth(); x++) { - setOutPixel(errors, out, in, x, y, 1); - } - if (++y >= in.getHeight()) { - break; - } - for (int x = in.getWidth()-1; x >= 0; x--) { - setOutPixel(errors, out, in, x, y, -1); - } - } - return out; - } - - private void setOutPixel(Errors errors, BufferedImage out, BufferedImage in, int x, int y, int dx) { - final Color color = new Color(in.getRGB(x, y)); - final Color adjustedColor = errors.adjust(color, x, y); - - final int newColor = getNearestColorAndDiffuseError(errors, - x, dx, y, - adjustedColor, color); - - out.setRGB(x, y, newColor); - } - - private BufferedImage createOutBuffer(BufferedImage in) { - - var indexModel = createIndexColorModel(); - - return new BufferedImage(indexModel, - indexModel.createCompatibleWritableRaster(in.getWidth(), in.getHeight()), - false, null); - - } - - private BufferedImage resize(BufferedImage src) { - if (maxWidth < 0 || maxHeight < 0) { - return src; - } - final int width = src.getWidth(); - final int height = src.getHeight(); - - double scaleF = Math.min(scaleFactor(width, maxWidth), - scaleFactor(height, maxHeight)); - - if (scaleF < 1.0) { - int newWidth = (int)Math.min(maxWidth, scaleF * width); - int newHeight = (int)Math.min(maxHeight, scaleF * height); - - return Scalr.resize(src, - Scalr.Method.QUALITY, - Scalr.Mode.AUTOMATIC, - newWidth, newHeight); - } - - return src; - } - - private double scaleFactor(int actualValue, int desiredValue) { - if (actualValue <= desiredValue) { - return 1.; - } - return desiredValue / (double) actualValue; - } - - private IndexColorModel createIndexColorModel() { - byte[] reds = new byte[palette.length]; - byte[] greens = new byte[palette.length]; - byte[] blues = new byte[palette.length]; - - for (int i = 0; i < palette.length; i++) { - int colorInt = palette[i].toInt(); - - reds[i] = (byte) ((colorInt >>> 16) & 0xFF); - greens[i] = (byte) ((colorInt >>> 8) & 0xFF); - blues[i] = (byte) ((colorInt) & 0xFF); - } - - return new IndexColorModel(getPaletteBits(palette), palette.length, reds, greens, blues); - - } - - private int getPaletteBits(Color[] palette) { - if (palette.length <= 16) { - return 4; - } - else { - return 8; - } - } - - private int getNearestColorAndDiffuseError(Errors errors, int x, int dx, int y, Color color, Color colorOrig) { - - var match = Arrays.stream(palette).min(Comparator.comparing(c -> c.delta(color))); - assert match.isPresent(); - - var retC = match.get(); - var error = colorOrig.minus(retC); - - errors.add(x+dx, y, error.scale(7/16.)); - errors.add(x+dx, y+1, error.scale(1/16.)); - errors.add(x, y+1, error.scale(5/16.)); - errors.add(x-dx, y+1, error.scale(3/16.)); - - return retC.toInt(); - } -} - -class Errors { - private final int width; - private final int height; - private final Color[] errors; - - Errors(int width, int height) { - this.width = width; - this.height = height; - - errors = new Color[width * height]; - } - - public void add(int x, int y, Color color) { - if (x > 0 && y > 0 && x + 1 < width && y + 1 < height) { - int index = getIndex(x, y); - if (errors[index] == null) { - errors[index] = color; - } - else { - errors[index] = errors[index].plus(color); - } - } - } - - public Color adjust(Color in, int x, int y) { - int idx = getIndex(x, y); - - if (errors[idx] != null) { - return in.plus(errors[idx]); - } - return in; - } - - private int getIndex(int x, int y) { - return x * height + y; - } -} - -@AllArgsConstructor -class Color { - private final double r; - private final double g; - private final double b; - - Color(int hex) { - this.b = ((hex) & 0xFF); - this.g = ((hex >>> 8) & 0xFF); - this.r = ((hex >>> 16) & 0xFF); - } - - int toInt() { - double bv = clampByteRange(b); - double gv = clampByteRange(g); - double rv = clampByteRange(r); - - return (((int)bv&0xFF) | (((int)gv & 0xFF) << 8) | (((int)rv & 0xFF) << 16)); - } - - double clampByteRange(double v) { - if (v < 0) return 0; - if (v > 255) return 255; - return v; - } - - public Color scale(double factor) { - return new Color(r*factor, g*factor, b*factor); - } - - public Color plus(Color other) { - return new Color(r+other.r, g+other.g, b+other.b); - } - - public Color minus(Color other) { - return new Color(r-other.r, g-other.g, b-other.b); - } - - public double delta(Color other) { - double avgr = (r + other.r)/2; - double dr = r - other.r; - double dg = g - other.g; - double db = b - other.b; - - if (avgr > 128) { - return Math.sqrt(2 * dr * dr + 4 * dg * dg + 3 * db * db); - } - else { - return Math.sqrt(3 * dr * dr + 4 * dg * dg + 2 * db * db); - } - } -} diff --git a/other/memex/src/main/java/nu/marginalia/memex/util/dithering/Palettes.java b/other/memex/src/main/java/nu/marginalia/memex/util/dithering/Palettes.java deleted file mode 100644 index a8137c9a..00000000 --- a/other/memex/src/main/java/nu/marginalia/memex/util/dithering/Palettes.java +++ /dev/null @@ -1,29 +0,0 @@ -package nu.marginalia.memex.util.dithering; - -public class Palettes { - - public static final int[] MARGINALIA_PALETTE = new int[] { - 0x000000, - 0x000000, - 0x808080, - 0x404040, - - 0xefefc0, - 0xf8f8ee, - 0x274fa5, - 0x85172f, - - 0x808060, - 0x60a060, - 0xFFFFFF, - }; - - public static int[] CGA_PALETTE = new int[]{ - 0x000000, 0xFFFFFF, 0x808080, 0xFF0000, - 0x800000, 0x00FF00, 0x008000, 0x0000FF, - 0x000080, 0xFFFF00, 0x808000, 0x00FFFF, - 0x008080, 0xFF00FF, 0x800080, 0x404040 - }; - - -} diff --git a/other/memex/src/main/java/nu/marginalia/util/FileSizeUtil.java b/other/memex/src/main/java/nu/marginalia/util/FileSizeUtil.java deleted file mode 100644 index de0cb17b..00000000 --- a/other/memex/src/main/java/nu/marginalia/util/FileSizeUtil.java +++ /dev/null @@ -1,18 +0,0 @@ -package nu.marginalia.util; - -public class FileSizeUtil { - public static String readableSize(long byteCount) { - if (byteCount < 1024L) { - return String.format("%db", byteCount); - } - if (byteCount < 1024*1024L) { - return String.format("%2.2fKb", byteCount/1024.); - } - if (byteCount < 1024*1024*1024L) { - return String.format("%2.2fMb", byteCount/1024/1024.); - } - - return String.format("%2.2fGb", byteCount/1024/1024L/1024.); - - } -} diff --git a/other/memex/src/main/resources/static/memex/ico/dir.png b/other/memex/src/main/resources/static/memex/ico/dir.png deleted file mode 100644 index c4bc7f53e6472e0fcdad8105f9115a7e87074f63..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 174 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|GCW-zLo9le zQxXyqUd(^@bv`4nNLpU9i}+!-h+oUA|DNxxYj)+j9H2AJA?N?!lW+b5f#)(w35kEk zh6V-(8$QfS`SJPYoJJr>nZ-HNnXh4YgA%h_tidZLuh;{J4gkTa@~#|ahW+kIE=C1k R^?-IVc)I$ztaD0e0st56LgxSg diff --git a/other/memex/src/main/resources/static/memex/ico/doc32.png b/other/memex/src/main/resources/static/memex/ico/doc32.png deleted file mode 100644 index 5b9f3731a8650210e1a9308e30fe3c60dc128cc1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 247 zcmeAS@N?(olHy`uVBq!ia0vp^3LwnE1|*BCs=fdz#^NA%Cx&(BWL^R}8$DedLn2z= zPPxc+Sb@hi-_5?tU;~fLyY=tq&AJ(C<+`}`gM%W+w&{~i4zMmP-20e~;T~(1I7frZ zc5b!OFEz{z2Od|ldv40|Jk+Lac4on~jJ{q&53k;AJ) zH@&ldWG`U#<%tMPRVcSR!+S+yw(*3$?7P@rWG4L3l9NmL$9nmIi2-j+vEbDSS@DJ3 vSsJ&)8{o>!YizxaRu<wHFDk+i&I7xBYv5x=gCz~JfX=d#Wzp$Pzl&t;$h diff --git a/other/memex/src/main/resources/static/memex/ico/nav16.png b/other/memex/src/main/resources/static/memex/ico/nav16.png deleted file mode 100644 index bda9f33f1c7ff0bbc876dc5027bd4a73408d7137..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 301 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbKJV{wqX6T`Z5GB1IggaDrqS0JsV zVB)G_lV)kyA7*{OQ0?Y|w0DPR{r+S&^JzfVJ_GY9dES}&LEb7xHkS7GB?bvi(RmM( zXaA2s`{eZPKmY&#pS^Z_OI>?rR#s$KP?=NL8Y82d;mOylclA&A>dH6xoxJBf&<=%? zAirP+plW R?_8kK44$rjF6*2UngBQthwK0V diff --git a/other/memex/src/main/resources/static/memex/ico/pic16.png b/other/memex/src/main/resources/static/memex/ico/pic16.png deleted file mode 100644 index 9c5a562e21212e64fad60cb434ab95a4043f4903..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 296 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbKJV{wqX6T`Z5GB1IggaDrqS0JsV zVB)G_lV)kyA7*{OQ0?Y|w0DPR{r+S&^JzfVJ_GY9dES}&LEb7xHkS7GB?bvi(RmM( zXaA2s`{eZPKmY&#pS^Z_OI>?rR#s$KP?=NL8Y82d;mOylclA&A>dH6xoxJBf&<=%? zAirP+plW6P)Px$l1W5CR9J;$Y^iJi&p;ZTkAV;$}!Q@W+YJESiyiT zv{~ro07gc5=>KP6V1RSz5(3qY6%7CX|7ZCBA2YepEedb}lqgthY)nu?@Y%CebpXjB z;NVa?unwTMBgjqz)N%w>v!TV*d=&nP?1=>RP%RgrTaHVNW)4USe1swP^Na{VeN>GC zW359d{9|W=2H92w;r@avecR8Zby``p>|CCl`?Fh%~A74B7h);e2DQLkv^1&ttQgfdSGc zV8H7Da==taObd~LVPZ3!ePKZzBh?*1q{R#j^lT9!TYi3iErvYJLV%FP7!E*g4N={Yi!K0*4P-s5!jk*=xP`l8DXiMpv7=KEp_ex(K{x@v<=Zg$hmWHHn!F; zvLhLg5+T+!KuQ!KI~JeCxB$h85aw8NJD3CBFBotKz!C%9y69;3G5`SoV|=4x1dZMR O0000?rR#s$KP?=NL8Y82d;mOylclA&A>dH6xoxJBf&<=%? zAirP+plWzIBxPg S#TfyOX7F_Nb6Mw<&;$Tn<%nef diff --git a/other/memex/src/main/resources/static/memex/ico/shiba16.png b/other/memex/src/main/resources/static/memex/ico/shiba16.png deleted file mode 100644 index e41d0f608fbcf7431e7a13fe264e1333eb8043e5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 316 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbKJV{wqX6T`Z5GB1IggaDrqS0JsV zVB)G_lV)kyA7*{OQ0?Y|w0DPR{r+S&^JzfVJ_GY9dES}&LEb7xHkS7GB?bvi(RmM( zXaA2s`{eZPKmY&#pS^Z_OI>?rR#s$KP?=NL8Y82d;mOylclA&A>dH6xoxJBf&<=%? zAirP+plW(di$YZHt%|EYY~r=evPMW=8HH;-BMfeFZSFQ hW~J(ot4!Z(8MAs>?rR#s$KP?=NL8Y82d;mOylclA&A>dH6xoxJBf&<=%? zAirP+plWgE{-7*Q(gNn@*Pp&V0n7h`8-Re)35*bw^BW|^R3TyZ9Dey z^{2996CIyC*=yc#a(3zS8+$zeEMMI4b?TAQo>S#Zm=^MIGfc>FFqF9bPy677EQS)M zvwY{%rM9X{U#!e5i|BYTS4H5%nw`7mEN3@c^WIqLgY!P0FUQ!0%edYzOIbbx=m-W+ LS3j3^P6 *:last-child { - background: #3F5F6F !important; - color: #fff; -} - -header nav a:hover, header nav a:focus { - background: #2f4858; - color: #fff !important; -} - -article a:focus { - color: #fff !important; - text-shadow: 0 0 5px #f00; - background-color: #000 !important;; - outline: none; -} -article { - max-width: 160ch; - margin-left: auto; - margin-right: auto; - display: flex; -} - -article > #memex-node { - line-height: 1.6; - margin: 1ch 2ch 2ch 2ch; - padding: 0ch 1ch 1ch 1ch; - flex-basis: 60ch; - background-color: #fff; - - border: 2px #ccc; - background-color: #fff; - border-left: 1px solid #ecb; - border-top: 1px solid #ecb; - - box-shadow: #0008 0 0 5px; - max-width: 60ch; - overflow: auto; -} - -pre { - font-size: 8pt; - overflow: auto; -} - -#memex-node h1, article #memex-node h2, article #memex-node h3 { - margin: 0px 0px 1ch 0px; - font-size: 12pt; - font-weight: normal; - margin-left: -1.6ch; - margin-right: -0.6ch; - background-color: #2f4858; - color: #fff; - padding: 0.75ch 1ch 0.5ch 1.5ch; - box-shadow: #000a 4px 4px; -} - -article a.doc:before { - width: 1ch; - height: 1ch; - content: url('/ico/file.png'); - margin-right: .5ch; -} -article a.dir:before { - width: 1ch; - height: 1ch; - content: url('/ico/dir.png'); - margin-right: .5ch; -} -article a.img:before { - width: 1ch; - height: 1ch; - content: url('/ico/pic16.png'); - margin-right: .5ch; -} -#sidebar > * { - padding: 1ch; -} -#sidebar h1 { - margin-top: 0; - font-size: 12pt; - font-weight: normal; - margin-left: -1.6ch; - margin-right: -0.6ch; - background-color: #2f4858; - color: #fff; - padding: 0.5ch 1ch 0.5ch 1ch; - box-shadow: #000a 4px 4px; -} - -article ul, article dl { - margin: 0px; -} -#sidebar a { - color: #000; -} - -#memex-node img { - width: 100%; - margin-left: auto; - margin-right: auto; -} - -.toc { - margin-top: 1ch; -} -.toc a { - display: block; -} - -footer { - padding: 2ch; - margin: 16ch 0px 0px 0px; - background-color: #acae89; - height: 20ch; - font-size: 10pt; -} -@media only screen and (max-device-width: 800px) { - header nav a { - padding: 1ch !important; - } - - .topbar { - flex-direction: column; - } - article { - flex-direction: column; - } - #memex-node { - flex-basis: unset !important; - } -} -@media (prefers-color-scheme: dark) { - a { - color: #acf; - } - header { - background-color: #000; - } - article #memex-node { - border: unset; - } - header nav a { - color: #eee; - } - nav.topbar { - background: #222; - box-shadow: #0008 0 0 5px; - } - nav.topbar a { - color: #eee; - } - nav.topbar h1 { - background-color: unset; - border-right: 2px solid #444; - border-left: none; - border-top: none; - border-bottom: none; - } - footer { - background-color: #000; - color: #fff; - } - body { - background-color: #444; - } - #sidebar { - color: #fff; - } - #sidebar a { - color: #ccc; - } - #memex-node { - background-color: #222 !important; - color: #aaa; - box-shadow: #0008 0 0 5px; - } -} - diff --git a/other/memex/src/main/resources/templates/auth/login.hdb b/other/memex/src/main/resources/templates/auth/login.hdb deleted file mode 100644 index dc78489c..00000000 --- a/other/memex/src/main/resources/templates/auth/login.hdb +++ /dev/null @@ -1,36 +0,0 @@ - - - - {{service}} - log in - - - - - - -
    - -
    -
    -
    -

    {{service}}: You must log in

    -

    - This is not a public-access system. If you do not already have - the password, it is not yours to know. -

    -

    I am never the less legally obliged to inform you that if you - were to log in, this would place a cookie on your computer. This - cookie would then be used to keep track of your status of having - logged in, and nothing else. -

    -
    - - - -
    -
    -
    - - diff --git a/other/memex/src/main/resources/templates/memex/memex-create-form.hdb b/other/memex/src/main/resources/templates/memex/memex-create-form.hdb deleted file mode 100644 index 6fbd6cec..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-create-form.hdb +++ /dev/null @@ -1,23 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -
    -

    New : {{url}}

    -
    -
    - -
    - - -
    -
    -

    Existing posts

    -
    -
      {{#each docs}}
    • {{url}}
      • {{/each}}
    -
    -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-delete-form.hdb b/other/memex/src/main/resources/templates/memex/memex-delete-form.hdb deleted file mode 100644 index ab41abcf..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-delete-form.hdb +++ /dev/null @@ -1,23 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -

    Delete {{url}}

    -
    -
    - - -
    -
    - -
    -
    -{{#if doc}}{{{doc}}}{{/if}} -{{#if image}}{{/if}} -
    -{{>memex/partial/memex-backlinks}} -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-image.hdb b/other/memex/src/main/resources/templates/memex/memex-image.hdb deleted file mode 100644 index 795d4f9c..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-image.hdb +++ /dev/null @@ -1,24 +0,0 @@ - - -{{>memex/partial/memex-head}} - - -{{>memex/partial/memex-topbar url=path}} - -
    -
    -

    {{path}}

    - -
    - -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-index-feed.hdb b/other/memex/src/main/resources/templates/memex/memex-index-feed.hdb deleted file mode 100644 index 352897bb..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-index-feed.hdb +++ /dev/null @@ -1,20 +0,0 @@ - - - marginalia.nu{{url}} - {{title}} - - marginalia kontakt@marginalia.nu - {{domain}}/ - {{now}} -{{#each docs}} -{{#amgarp DRAFT}} - - {{title}} - - {{domain}}{{url}} - {{date}}T00:00:00Z - -{{/amgarp}} -{{/each}} - - \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/memex-index.hdb b/other/memex/src/main/resources/templates/memex/memex-index.hdb deleted file mode 100644 index 59243c33..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-index.hdb +++ /dev/null @@ -1,34 +0,0 @@ - - -{{>memex/partial/memex-head}} - - -{{>memex/partial/memex-topbar}} - -
    - -
    -{{#if indexData}}{{{indexData}}}{{/if}} -{{#unless indexData}}

    {{url}}

    {{/unless}} - -{{>memex/partial/memex-task-listing}} -{{>memex/partial/memex-documents-inline}} -
    - - -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-rename-form.hdb b/other/memex/src/main/resources/templates/memex/memex-rename-form.hdb deleted file mode 100644 index db969027..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-rename-form.hdb +++ /dev/null @@ -1,23 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -

    Rename {{url}}

    -
    -
    - - -
    -
    - -
    -
    -{{#if doc}}{{{doc}}}{{/if}} -{{#if image}}{{/if}} -
    -{{>memex/partial/memex-backlinks}} -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-tombstone.hdb b/other/memex/src/main/resources/templates/memex/memex-tombstone.hdb deleted file mode 100644 index 905b1bec..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-tombstone.hdb +++ /dev/null @@ -1,17 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -
    -

    {{url}} is gone

    -

    -{{#if message}}{{{message}}}{{/if}} -{{#if redirect}}See {{redirect}}{{/if}} -

    -
    -{{>memex/partial/memex-backlinks}} -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-update-form.hdb b/other/memex/src/main/resources/templates/memex/memex-update-form.hdb deleted file mode 100644 index 6b7a55e5..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-update-form.hdb +++ /dev/null @@ -1,20 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -
    -
    - -
    - - -
    -
    - -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-upload-form.hdb b/other/memex/src/main/resources/templates/memex/memex-upload-form.hdb deleted file mode 100644 index e503a2db..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-upload-form.hdb +++ /dev/null @@ -1,26 +0,0 @@ - - -{{>memex/partial/memex-head}} - -{{>memex/partial/memex-topbar}} -
    -
    -

    Upload : {{url}}

    -
    -
    - -
    -
    -
    -
    - -
    -
    -

    Existing posts

    -
    -
      {{#each docs}}
    • {{url}}
      • {{/each}} -
        {{#each images}}
      • {{url}}
        • {{/each}} -
    -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/memex-view.hdb b/other/memex/src/main/resources/templates/memex/memex-view.hdb deleted file mode 100644 index 47eb1480..00000000 --- a/other/memex/src/main/resources/templates/memex/memex-view.hdb +++ /dev/null @@ -1,36 +0,0 @@ - - -{{>memex/partial/memex-head}} - - -{{>memex/partial/memex-topbar url=baseDoc.url}} - -
    -
    -{{{doc}}} - -{{#pragma this "TOPIC"}} -{{>memex/partial/memex-backlinks-inline}} -{{/pragma}} -
    - -
    -{{> memex/partial/memex-footer}} - diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-backlinks-inline.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-backlinks-inline.hdb deleted file mode 100644 index de40e40d..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-backlinks-inline.hdb +++ /dev/null @@ -1,6 +0,0 @@ -{{#if backlinks}} -
    {{#each backlinks}} -
    {{url}}
    -
    {{description}}
    -{{/each}}
    -{{/if}} \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-backlinks.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-backlinks.hdb deleted file mode 100644 index a15d7710..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-backlinks.hdb +++ /dev/null @@ -1,9 +0,0 @@ -{{#if backlinks}} - -{{/if}} \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-directories.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-directories.hdb deleted file mode 100644 index 164707c5..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-directories.hdb +++ /dev/null @@ -1,9 +0,0 @@ -
    -

    Directories

    -{{#if parent}} - ..
    -{{/if}} -{{#each directories}} - {{filename}}
    -{{/each}} -
    \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-documents-inline.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-documents-inline.hdb deleted file mode 100644 index 59598460..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-documents-inline.hdb +++ /dev/null @@ -1,10 +0,0 @@ -{{#pragma this "LISTING"}} -{{#if docs}}

    Documents


    {{/if}} -
    -{{#each docs}} -{{#unless index}} - {{url.filename}}
    -{{/unless}} -{{/each}} -
    -{{/pragma}} \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-documents.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-documents.hdb deleted file mode 100644 index 242004ac..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-documents.hdb +++ /dev/null @@ -1,10 +0,0 @@ -{{#amgarp this "LISTING"}} -
    -{{#if docs}}

    Documents

    {{/if}} -{{#each docs}} -{{#unless index}} - {{url.filename}}
    -{{/unless}} -{{/each}} -
    -{{/amgarp}} \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-footer.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-footer.hdb deleted file mode 100644 index 32edef62..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-footer.hdb +++ /dev/null @@ -1,4 +0,0 @@ - \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-head.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-head.hdb deleted file mode 100644 index af30c6f5..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-head.hdb +++ /dev/null @@ -1,9 +0,0 @@ - - - MEMEX - {{title}} - - - {{#pragma this "FEED"}} - - {{/pragma}} - \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-images.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-images.hdb deleted file mode 100644 index 6498d04e..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-images.hdb +++ /dev/null @@ -1,6 +0,0 @@ -
    -{{#if images}}

    Images

    {{/if}} -{{#each images}} - {{path.filename}}
    -{{/each}} -
    \ No newline at end of file diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-task-listing.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-task-listing.hdb deleted file mode 100644 index 6f9b8060..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-task-listing.hdb +++ /dev/null @@ -1,10 +0,0 @@ -{{#if tasks}} -
    -

    Open Tasks

    -
    -{{#each tasks}} -
    {{task}}
    -{{/each}} -
    -
    -{{/if}} diff --git a/other/memex/src/main/resources/templates/memex/partial/memex-topbar.hdb b/other/memex/src/main/resources/templates/memex/partial/memex-topbar.hdb deleted file mode 100644 index 940ec5bb..00000000 --- a/other/memex/src/main/resources/templates/memex/partial/memex-topbar.hdb +++ /dev/null @@ -1,13 +0,0 @@ -
    - -
    - \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/gmi/GemtextDatabaseTest.java b/other/memex/src/test/java/nu/marginalia/gmi/GemtextDatabaseTest.java deleted file mode 100644 index 3a9cf4dd..00000000 --- a/other/memex/src/test/java/nu/marginalia/gmi/GemtextDatabaseTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.gmi; - -import nu.marginalia.memex.gemini.gmi.GemtextDatabase; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.Optional; - -class GemtextDatabaseTest { - - @Test - public void test() { - var db = new GemtextDatabase(new MemexNodeUrl("/test.gmi"), new String[] { - "=> / foo", - "=> /x bar", - "=> /y baz", - "=> /z" - }); - verifyResult("foo", db.getLinkData(new MemexNodeUrl("/"))); - verifyResult("bar", db.getLinkData(new MemexNodeUrl("/x"))); - verifyResult("baz", db.getLinkData(new MemexNodeUrl("/y"))); - verifyResult("", db.getLinkData(new MemexNodeUrl("/z"))); - Assertions.assertFalse(db.getLinkData(new MemexNodeUrl("/w")).isPresent()); - } - - void verifyResult(String expected, @SuppressWarnings("OptionalUsedAsFieldOrParameterType") Optional actual) { - Assertions.assertTrue(actual.isPresent(), () -> "No value found, expected " + expected); - Assertions.assertEquals(expected, actual.get()); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/gmi/GemtextDocumentTest.java b/other/memex/src/test/java/nu/marginalia/gmi/GemtextDocumentTest.java deleted file mode 100644 index 1edc5832..00000000 --- a/other/memex/src/test/java/nu/marginalia/gmi/GemtextDocumentTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package nu.marginalia.gmi; - -import nu.marginalia.memex.gemini.gmi.GemtextDatabase; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.memex.gemini.gmi.line.GemtextLink; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.model.MemexUrl; -import org.junit.jupiter.api.Test; - -import java.util.Arrays; - -import static org.junit.jupiter.api.Assertions.*; - -class GemtextDocumentTest { - - @Test - void testEmpty() { - var document = GemtextDocument.of(new MemexNodeUrl("/test.gmi"), ""); - assertEquals("/test.gmi", document.getTitle()); - assertTrue(document.getLinks().isEmpty()); - } - - @Test - void testParseTombstone() { - var lines = new String[] { - "# Tombstone", -"", - "This special file contains information about removed resources.", -"", -"# Removed links", -"=> /dead.gmi It was never here, I swear", - "=> /dead2.png Removed through an act of God", - "=> /worklog.gmi Old and unused file.", - "=> /todo.gmi Empty file", - "=> /search-about.gmi Confusingly gemini-specific", - "=> /05-test.gmi Cursed testing file"}; - - var document = GemtextDatabase.of(new MemexNodeUrl("/test.gmi"), lines); - Arrays.stream(document.getLines()).forEach(System.out::println); - document.keys().forEach(k -> System.out.println(k + "-" + document.getLinkData(new MemexNodeUrl(k)).orElse(""))); - -} - @Test - void testVanilla() { - var document = GemtextDocument.of(new MemexNodeUrl("/test.gmi"), - "# Test Document", - "=> /foo.gmi\tMy foos", - "=>/bar.gmi\tMy bars", - "=>/baz.gmi", - "=>/foobar.gmi ", - "=>/volvo240.png hey cool car right", - "=>", - "=> ", - " => ", - "## Goodbye", - "... and good luck"); - assertEquals("Test Document", document.getTitle()); - Arrays.stream(document.getLines()).forEach(System.out::println); - assertEquals(5, document.getLinks().size()); - document.getLinks().forEach(System.out::println); - - assertArrayEquals(new String[] { - "/foo.gmi", "/bar.gmi", "/baz.gmi", "/foobar.gmi", "/volvo240.png" - }, - document.getLinks().stream().map(GemtextLink::getUrl).map(MemexUrl::getUrl).toArray()); - - assertArrayEquals(new String[] {"My foos", "My bars", null, null, "hey cool car right"}, - document.getLinks().stream().map(GemtextLink::getTitle).toArray()); - } - - - - @Test - void testTasks() { - var document = GemtextDocument.of(new MemexNodeUrl("/test.gmi"), - "# Test Document", - "- Go shopping", - "-- Milk", - "-- Eggs", - "-- Bacon", - "--- If they have organic, buy two", - "- Go dancing", - "Stuff", - "- Go dancing again"); - assertEquals("Test Document", document.getTitle()); - Arrays.stream(document.getLines()).forEach(System.out::println); - document.getOpenTopTasks().values().forEach(System.out::println); - } - -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/gmi/parser/GemtextTaskParserTest.java b/other/memex/src/test/java/nu/marginalia/gmi/parser/GemtextTaskParserTest.java deleted file mode 100644 index d63d5f4c..00000000 --- a/other/memex/src/test/java/nu/marginalia/gmi/parser/GemtextTaskParserTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package nu.marginalia.gmi.parser; - -import nu.marginalia.memex.gemini.gmi.parser.GemtextTaskParser; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeTaskId; -import org.junit.jupiter.api.Test; - -class GemtextTaskParserTest { - - @Test - void parse() { - System.out.println(GemtextTaskParser.parse("-task", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - System.out.println(GemtextTaskParser.parse("- task", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - System.out.println(GemtextTaskParser.parse("--task", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - System.out.println(GemtextTaskParser.parse("-task(/)", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - System.out.println(GemtextTaskParser.parse("-task(-)", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - System.out.println(GemtextTaskParser.parse("-task(?)(x)", new MemexNodeHeadingId(0), new MemexNodeTaskId(0))); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/dithering/FloydSteinbergDitherTest.java b/other/memex/src/test/java/nu/marginalia/memex/dithering/FloydSteinbergDitherTest.java deleted file mode 100644 index 7d18977e..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/dithering/FloydSteinbergDitherTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package nu.marginalia.memex.dithering; - -import nu.marginalia.memex.util.dithering.FloydSteinbergDither; -import nu.marginalia.memex.util.dithering.Palettes; -import org.junit.jupiter.api.Test; - -import javax.imageio.ImageIO; -import java.io.File; -import java.io.IOException; - -class FloydSteinbergDitherTest { - - @Test - public void test() throws IOException { -// convert("/home/vlofgren/Work/dither/volvo.jpg", "/home/vlofgren/Work/dither/volvo-raster.png"); -// convert("/home/vlofgren/Work/dither/dog.jpg", "/home/vlofgren/Work/dither/dog-raster.png"); -// convert("/home/vlofgren/Work/dither/robocop.jpg", "/home/vlofgren/Work/dither/robocop-raster.png"); -// convert("/home/vlofgren/Work/dither/socrates.jpeg", "/home/vlofgren/Work/dither/socrates-raster.png"); - - -// convert("C:\\Users\\vlofg\\Documents\\volvo.jpg", -// "C:\\Users\\vlofg\\Documents\\volvo-raster.png"); -// convert("C:\\Users\\vlofg\\Documents\\socrates.jpg", -// "C:\\Users\\vlofg\\Documents\\socrates-raster.png"); -// convert("C:\\Users\\vlofg\\Documents\\goya_nude_maja.jpg", -// "C:\\Users\\vlofg\\Documents\\goya_nude_maja-raster.png"); - } - - void convert(String in, String out) throws IOException { - var result = new FloydSteinbergDither(Palettes.MARGINALIA_PALETTE, 640, 480).convert(ImageIO.read(new File(in))); - - ImageIO.write(result, "png", new File(out)); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/MemexFileWriterTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/MemexFileWriterTest.java deleted file mode 100644 index 32833242..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/MemexFileWriterTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package nu.marginalia.memex.memex; - -import nu.marginalia.util.test.TestUtil; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.*; - -class MemexFileWriterTest { - - Path root; - MemexFileWriter renderedResources; - @BeforeEach - void setUp() throws IOException { - root = Files.createTempDirectory(getClass().getSimpleName()); - renderedResources = new MemexFileWriter(root); - } - - @AfterEach - void tearDown() { - TestUtil.clearTempDir(root); - } - - @Test - void exists() throws IOException { - assertFalse(renderedResources.exists(new MemexNodeUrl("/test"))); - renderedResources.write(new MemexNodeUrl("/test"), "A line"); - assertTrue(renderedResources.exists(new MemexNodeUrl("/test"))); - } - - @Test - void write() throws IOException { - renderedResources.write(new MemexNodeUrl("/test"), "A line"); - assertEquals("A line", Files.readString(root.resolve("test"))); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/MemexTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/MemexTest.java deleted file mode 100644 index 54d80069..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/MemexTest.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.memex.memex; - -import org.junit.jupiter.api.Test; - -import java.io.IOException; - -class MemexTest { - @Test - public void test() { - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextChangeTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextChangeTest.java deleted file mode 100644 index d6548042..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextChangeTest.java +++ /dev/null @@ -1,275 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.GeminiServiceImpl; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.MemexData; -import nu.marginalia.memex.memex.MemexLoader; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.renderer.MemexRendererers; -import nu.marginalia.memex.memex.system.MemexFileSystemModifiedTimes; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.memex.system.MemexSourceFileSystem; -import nu.marginalia.memex.memex.system.git.MemexGitRepoImpl; -import nu.marginalia.util.test.TestUtil; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - - -class GemtextChangeTest { - - - private Memex memex; - private Path tempDir; - - private final String tombstonePath = "/special/tombstone.gmi"; - private final String redirectPath = "/special/redirects.gmi"; - private final String testFilePath = "/test.gmi"; - - static final Logger logger = LoggerFactory.getLogger(GemtextChangeTest.class); - - @BeforeAll - public static void init() { - - RxJavaPlugins.setErrorHandler(e -> { - if (e.getMessage() == null) { - logger.error("Error", e); - } - else { - logger.error("Error {}: {}", e.getClass().getSimpleName(), e.getMessage()); - } - }); - } - - @SneakyThrows - @BeforeEach - public void setUp() { - tempDir = Files.createTempDirectory("test"); - Files.createDirectory(tempDir.resolve("special")); - var data = new MemexData(); - - memex = new Memex(data, null, - Mockito.mock(MemexGitRepoImpl.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(), - new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)), - tempDir, tombstonePath, redirectPath), - Mockito.mock(MemexFileWriter.class), - null, - Mockito.mock(MemexRendererers.class), - Mockito.mock(GeminiServiceImpl.class)); - } - - @SneakyThrows - @AfterEach - public void tearDown() { - TestUtil.clearTempDir(tempDir); - } - - @Test - void appendHeading() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { "3"}).visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(5, lines.size()); - assertEquals("# Header", lines.get(0)); - assertEquals("1", lines.get(1)); - assertEquals("## Header 2", lines.get(2)); - assertEquals("3", lines.get(3)); - assertEquals("# Header 3", lines.get(4)); - } - - @Test - void appendRoot() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextAppend(url, new MemexNodeHeadingId(0), new String[] { "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(5, lines.size()); - assertEquals("# Header", lines.get(0)); - assertEquals("1", lines.get(1)); - assertEquals("## Header 2", lines.get(2)); - assertEquals("# Header 3", lines.get(3)); - assertEquals("3", lines.get(4)); - } - - @Test - void appendMissing() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextAppend(url, new MemexNodeHeadingId(5), new String[] { "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(5, lines.size()); - assertEquals("# Header", lines.get(0)); - assertEquals("1", lines.get(1)); - assertEquals("## Header 2", lines.get(2)); - assertEquals("# Header 3", lines.get(3)); - assertEquals("3", lines.get(4)); - } - - @Test - void replaceHeading() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextReplace(url, new MemexNodeHeadingId(1), new String[] { "# New", "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(3, lines.size()); - assertEquals("# New", lines.get(0)); - assertEquals("3", lines.get(1)); - assertEquals("# Header 3", lines.get(2)); - } - - @Test - void replaceRoot() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextReplace(url, new MemexNodeHeadingId(0), new String[] { "# New", "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(2, lines.size()); - assertEquals("# New", lines.get(0)); - assertEquals("3", lines.get(1)); - } - - @Test - void replaceMissing() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "## Header 2", "# Header 3" - }) - ).visit(memex); - new GemtextReplace(url, new MemexNodeHeadingId(5), new String[] { "# New", "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(7, lines.size()); - - - assertEquals("# Header", lines.get(0)); - assertEquals("1", lines.get(1)); - assertEquals("## Header 2", lines.get(2)); - assertEquals("# Header 3", lines.get(3)); - - assertEquals("# Error! Replace failed!", lines.get(4)); - assertEquals("# New", lines.get(5)); - assertEquals("3", lines.get(6)); - } - - @Test - void prependHeading() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "2" - }) - ).visit(memex); - new GemtextPrepend(url, new MemexNodeHeadingId(1), new String[] { "3"}) - .visit(memex); - - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - lines.forEach(System.out::println); - assertEquals(4, lines.size()); - assertEquals("# Header", lines.get(0)); - assertEquals("3", lines.get(1)); - assertEquals("1", lines.get(2)); - assertEquals("2", lines.get(3)); - } - - @Test - void prependRoot() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "2" - }) - ).visit(memex); - new GemtextPrepend(url, new MemexNodeHeadingId(0), new String[] { "3" }) - .visit(memex); - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - - lines.forEach(System.out::println); - assertEquals(4, lines.size()); - assertEquals("3", lines.get(0)); - assertEquals("# Header", lines.get(1)); - assertEquals("1", lines.get(2)); - assertEquals("2", lines.get(3)); - } - - - @Test - void prependMissing() throws IOException { - var url = new MemexNodeUrl(testFilePath); - new GemtextCreateOrMutate(url, "# Header", - new GemtextAppend(url, new MemexNodeHeadingId(1), new String[] { - "1", "2" - }) - ).visit(memex); - new GemtextPrepend(url, new MemexNodeHeadingId(5), new String[] { "3" }) - .visit(memex); - - List lines = Files.readAllLines(Path.of(tempDir + testFilePath)); - - lines.forEach(System.out::println); - assertEquals(4, lines.size()); - assertEquals("# Header", lines.get(0)); - assertEquals("1", lines.get(1)); - assertEquals("2", lines.get(2)); - assertEquals("3", lines.get(3)); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTaskUpdateTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTaskUpdateTest.java deleted file mode 100644 index 06d74be4..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTaskUpdateTest.java +++ /dev/null @@ -1,227 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.GeminiServiceImpl; -import nu.marginalia.memex.gemini.gmi.GemtextDocument; -import nu.marginalia.util.test.TestUtil; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.MemexData; -import nu.marginalia.memex.memex.MemexLoader; -import nu.marginalia.memex.memex.change.update.GemtextDocumentUpdateCalculator; -import nu.marginalia.memex.memex.model.MemexNodeHeadingId; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.renderer.MemexRendererers; -import nu.marginalia.memex.memex.system.MemexFileSystemModifiedTimes; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.memex.system.MemexSourceFileSystem; -import nu.marginalia.memex.memex.system.git.MemexGitRepoImpl; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.Arrays; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class GemtextTaskUpdateTest { - - - private Memex memex; - private Path tempDir; - - private final String tombstonePath = "/special/tombstone.gmi"; - private final String redirectPath = "/special/redirects.gmi"; - private final String testFilePath = "/test.gmi"; - private final String todoFilePath = "/todo.gmi"; - private final String doneFilePath = "/done.gmi"; - - static final Logger logger = LoggerFactory.getLogger(GemtextTaskUpdateTest.class); - - @BeforeAll - public static void init() { - - RxJavaPlugins.setErrorHandler(e -> { - if (e.getMessage() == null) { - logger.error("Error", e); - } - else { - logger.error("Error {}: {}", e.getClass().getSimpleName(), e.getMessage()); - } - }); - } - - @SneakyThrows - @BeforeEach - public void setUp() { - tempDir = Files.createTempDirectory("test"); - Files.createDirectory(tempDir.resolve("special")); - var data = new MemexData(); - - memex = new Memex(data, null, Mockito.mock(MemexGitRepoImpl.class), new MemexLoader(data, new MemexFileSystemModifiedTimes(), - new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)), tempDir, tombstonePath, redirectPath), - Mockito.mock(MemexFileWriter.class), - null, - Mockito.mock(MemexRendererers.class), - Mockito.mock(GeminiServiceImpl.class)); - } - - @SneakyThrows - @AfterEach - public void tearDown() { - TestUtil.clearTempDir(tempDir); - } - - @Test - void updateTodoFileWithTodoTask() throws IOException { - - var url = new MemexNodeUrl(testFilePath); - - GemtextMutation.createOrAppend(url, "%%%TASKS\n# Header", new MemexNodeHeadingId(1), - "%%% TASKS", "## Todo", "- A task yet finished").visit(memex); - - GemtextDocumentUpdateCalculator updateCalculator = new GemtextDocumentUpdateCalculator(memex); - var updates = updateCalculator.calculateUpdates(memex.getDocument(url), MemexNodeHeadingId.ROOT, - GemtextDocument.of(url, "%%% TASKS", "# Header", "## Todo", "- A task yet finished (?)", "- One More Task")); - updates.forEach(System.out::println); - for (var update : updates) { - update.visit(memex); - } - - verifyFile(testFilePath, - "%%% TASKS", - "# Header", - "## Todo", - "- A task yet finished (?)", - "- One More Task" - ); - } - - @Test - void updateDoneFileWithTodoTask() throws IOException { - - var url = new MemexNodeUrl(testFilePath); - - GemtextMutation.createOrAppend(url, "%%% TASKS\n# Header", new MemexNodeHeadingId(1), - "## Done", "- A task yet finished (/)").visit(memex); - - GemtextDocumentUpdateCalculator updateCalculator = new GemtextDocumentUpdateCalculator(memex); - var updates = updateCalculator.calculateUpdates(memex.getDocument(url), MemexNodeHeadingId.ROOT, - GemtextDocument.of(url, "%%% TASKS", "# Header", "## Done", "- A task yet finished (?)")); - updates.forEach(System.out::println); - for (var update : updates) { - update.visit(memex); - } - - verifyFile(testFilePath, - "%%% TASKS", - "# Header", - "## Done" - ); - - verifyFile(todoFilePath, - "%%% TASKS", - "# Todo", - "- A task yet finished (?)" - ); - } - - @Test - void moveToDoneNewDoneFile() throws IOException { - - var url = new MemexNodeUrl(testFilePath); - - GemtextMutation.createOrAppend(url, "%%% TASKS\n# Header", new MemexNodeHeadingId(1), - "%%% TASKS","## Todo", "- A task yet finished").visit(memex); - - GemtextDocumentUpdateCalculator updateCalculator = new GemtextDocumentUpdateCalculator(memex); - var updates = updateCalculator.calculateUpdates(memex.getDocument(url), MemexNodeHeadingId.ROOT, - GemtextDocument.of(url, "%%% TASKS", "# Header", "## Todo", "- A task yet finished (/)")); - updates.forEach(System.out::println); - for (var update : updates) { - update.visit(memex); - } - - verifyFile(doneFilePath, - "%%% TASKS", - "# Done", - "", - "## Done " + LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE), - "- A task yet finished (/)"); - - updates = updateCalculator.calculateUpdates(memex.getDocument(url), MemexNodeHeadingId.ROOT, - GemtextDocument.of(url, "%%% TASKS", "# Header", "## Todo", "- Another task yet finished (/)")); - updates.forEach(System.out::println); - for (var update : updates) { - update.visit(memex); - } - - verifyFile(doneFilePath, - "%%% TASKS", - "# Done", - "", - "## Done " + LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE), - "- Another task yet finished (/)", - "- A task yet finished (/)"); - } - - @Test - void moveToDoneOldDoneFile() throws IOException { - - var doneUrl = new MemexNodeUrl(doneFilePath); - var url = new MemexNodeUrl(testFilePath); - - - GemtextMutation.createOrAppend(doneUrl, "%%% TASKS\n# Done", new MemexNodeHeadingId(1), - "## Done 2012-04-30", "- A very old task (/)").visit(memex); - - GemtextMutation.createOrAppend(url, "%%% TASKS\n# Header", new MemexNodeHeadingId(1), - "## Todo", "- A task yet finished").visit(memex); - - GemtextDocumentUpdateCalculator updateCalculator = new GemtextDocumentUpdateCalculator(memex); - var updates = updateCalculator.calculateUpdates(memex.getDocument(url), MemexNodeHeadingId.ROOT, - GemtextDocument.of(url, "%%% TASKS", "# Header", "## Todo", "- A task yet finished (/)")); - updates.forEach(System.out::println); - for (var update : updates) { - update.visit(memex); - } - - verifyFile(doneFilePath, - "%%% TASKS", - "# Done", - "", - "## Done " + LocalDate.now().format(DateTimeFormatter.ISO_LOCAL_DATE), - "- A task yet finished (/)", - "## Done 2012-04-30", - "- A very old task (/)"); - - } - - public void verifyFile(String file, String... lines) throws IOException { - Path p = Path.of(tempDir + file); - assertTrue(Files.exists(p), "File " + file + " is missing"); - List actualLines = Files.readAllLines(p); - System.out.println("Expecting: "); - Arrays.stream(lines).forEach(System.out::println); - System.out.println("Got: "); - actualLines.forEach(System.out::println); - System.out.println("-- end -- "); - - assertEquals(lines.length, actualLines.size()); - for (int i = 0; i < lines.length; i++) { - assertEquals(lines[i], actualLines.get(i)); - } - } - -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulatorTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulatorTest.java deleted file mode 100644 index 80e0b627..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/change/GemtextTombstoneUpdateCaclulatorTest.java +++ /dev/null @@ -1,107 +0,0 @@ -package nu.marginalia.memex.memex.change; - -import io.reactivex.rxjava3.plugins.RxJavaPlugins; -import lombok.SneakyThrows; -import nu.marginalia.memex.gemini.GeminiServiceImpl; -import nu.marginalia.util.test.TestUtil; -import nu.marginalia.memex.memex.Memex; -import nu.marginalia.memex.memex.MemexData; -import nu.marginalia.memex.memex.MemexLoader; -import nu.marginalia.memex.memex.model.MemexNodeUrl; -import nu.marginalia.memex.memex.renderer.MemexRendererers; -import nu.marginalia.memex.memex.system.MemexFileSystemModifiedTimes; -import nu.marginalia.memex.memex.system.MemexFileWriter; -import nu.marginalia.memex.memex.system.MemexSourceFileSystem; -import nu.marginalia.memex.memex.system.git.MemexGitRepoImpl; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; - - -class GemtextTombstoneUpdateCaclulatorTest { - - private GemtextTombstoneUpdateCaclulator updateCaclulator; - private Memex memex; - private Path tempDir; - - private final String tombstonePath = "/special/tombstone.gmi"; - private final String redirectPath = "/special/redirects.gmi"; - - static final Logger logger = LoggerFactory.getLogger(GemtextTombstoneUpdateCaclulatorTest.class); - - @BeforeAll - public static void init() { - - RxJavaPlugins.setErrorHandler(e -> { - if (e.getMessage() == null) { - logger.error("Error", e); - } - else { - logger.error("Error {}: {}", e.getClass().getSimpleName(), e.getMessage()); - } - }); - } - - @SneakyThrows - @BeforeEach - public void setUp() { - tempDir = Files.createTempDirectory("test"); - Files.createDirectory(tempDir.resolve("special")); - - updateCaclulator = new GemtextTombstoneUpdateCaclulator( - tombstonePath, - redirectPath - ); - var data = new MemexData(); - - memex = new Memex(data, null, - Mockito.mock(MemexGitRepoImpl.class), - new MemexLoader(data, new MemexFileSystemModifiedTimes(), - new MemexSourceFileSystem(tempDir, Mockito.mock(MemexGitRepoImpl.class)), tempDir, tombstonePath, redirectPath), - Mockito.mock(MemexFileWriter.class), - updateCaclulator, - Mockito.mock(MemexRendererers.class), - Mockito.mock(GeminiServiceImpl.class)); - } - - @SneakyThrows - @AfterEach - public void tearDown() { - TestUtil.clearTempDir(tempDir); - } - - @Test - void addTombstone() throws IOException { - updateCaclulator.addTombstone(new MemexNodeUrl("/deleted.gmi"), "It's gone jimmy").visit(memex); - updateCaclulator.addTombstone(new MemexNodeUrl("/deleted2.gmi"), "RIP").visit(memex); - List lines = Files.readAllLines(Path.of(tempDir + tombstonePath)); - assertEquals(3, lines.size()); - - assertEquals("# Tombstones", lines.get(0)); - assertEquals("=> /deleted.gmi\tIt's gone jimmy", lines.get(1)); - assertEquals("=> /deleted2.gmi\tRIP", lines.get(2)); - } - - @Test - void addRedirect() throws IOException { - updateCaclulator.addRedirect(new MemexNodeUrl("/deleted.gmi"), "/new").visit(memex); - updateCaclulator.addRedirect(new MemexNodeUrl("/deleted2.gmi"), "/new2").visit(memex); - List lines = Files.readAllLines(Path.of(tempDir + redirectPath)); - - assertEquals(3, lines.size()); - assertEquals("# Redirects", lines.get(0)); - assertEquals("=> /deleted.gmi\t/new", lines.get(1)); - assertEquals("=> /deleted2.gmi\t/new2", lines.get(2)); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/memex/memex/model/MemexNodeHeadingIdTest.java b/other/memex/src/test/java/nu/marginalia/memex/memex/model/MemexNodeHeadingIdTest.java deleted file mode 100644 index 73506861..00000000 --- a/other/memex/src/test/java/nu/marginalia/memex/memex/model/MemexNodeHeadingIdTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package nu.marginalia.memex.memex.model; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -class MemexNodeHeadingIdTest { - @Test - public void test() { - var heading = new MemexNodeHeadingId(0); - assertEquals("0", heading.toString()); - assertEquals("1", heading.next(0).toString()); - assertEquals("0.1", heading.next(1).toString()); - assertEquals("0.0.1", heading.next(2).toString()); - assertEquals("0.1", heading.next(2).next(1).toString()); - } - - @Test - public void testParenthood() { - var heading = new MemexNodeHeadingId(1,0,2); - - assertTrue(heading.isChildOf(new MemexNodeHeadingId(1,0))); - assertTrue(heading.isChildOf(new MemexNodeHeadingId(1))); - assertFalse(heading.isChildOf(new MemexNodeHeadingId(1,1))); - assertFalse(heading.isChildOf(new MemexNodeHeadingId(1,0,1))); - } - - @Test - public void testComparator() { - assertTrue(new MemexNodeHeadingId(1).compareTo(new MemexNodeHeadingId(2)) < 0); - assertTrue(new MemexNodeHeadingId(1).compareTo(new MemexNodeHeadingId(1, 1)) > 0); - } -} \ No newline at end of file diff --git a/other/memex/src/test/java/nu/marginalia/util/test/TestUtil.java b/other/memex/src/test/java/nu/marginalia/util/test/TestUtil.java deleted file mode 100644 index c8f3735a..00000000 --- a/other/memex/src/test/java/nu/marginalia/util/test/TestUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -package nu.marginalia.util.test; - - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; - -public class TestUtil { - private static boolean isTempDir(Path dir) { - return dir.startsWith("/tmp") || dir.toString().contains("tmp"); - } - - public static void clearTempDir(Path dir) { - if (!isTempDir(dir)) { - throw new IllegalArgumentException("Refusing to recursively delete directory with that name"); - } - if (Files.isDirectory(dir)) { - for (File f : dir.toFile().listFiles()) { - File[] files = f.listFiles(); - if (files != null) { - Arrays.stream(files).map(File::toPath).forEach(TestUtil::clearTempDir); - } - System.out.println("Deleting " + f + " (" + fileSize(f.toPath()) + ")"); - f.delete(); - } - } - System.out.println("Deleting " + dir); - dir.toFile().delete(); - } - - private static String fileSize(Path path) { - try { - long sizeBytes = Files.size(path); - - if (sizeBytes > 1024 * 1024 * 1024) return round(sizeBytes / 1073741824.) + "Gb"; - if (sizeBytes > 1024 * 1024) return round(sizeBytes / 1048576.) + "Mb"; - if (sizeBytes > 1024) return round(sizeBytes / 1024.) + "Kb"; - return sizeBytes + "b"; - } - catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - private static String round(double d) { - return String.format("%.2f", d); - } -} diff --git a/other/readme.md b/other/readme.md deleted file mode 100644 index 78f2b3cf..00000000 --- a/other/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# Other - -This code will be moved to a separate repository. - -Nothing to see here, move along. \ No newline at end of file diff --git a/other/wmsa_old/build.gradle b/other/wmsa_old/build.gradle deleted file mode 100644 index 4f38b40c..00000000 --- a/other/wmsa_old/build.gradle +++ /dev/null @@ -1,112 +0,0 @@ -plugins { - id 'java' - id "io.freefair.lombok" version "5.3.3.3" - - id 'jvm-test-suite' -} - -repositories { - mavenLocal() - maven { url "https://artifactory.cronapp.io/public-release/" } - maven { url "https://repo1.maven.org/maven2/" } - maven { url "https://www2.ph.ed.ac.uk/maven2/" } - maven { url "https://jitpack.io/" } - exclusiveContent { - forRepository { - maven { - url = uri("https://jitpack.io") - } - } - filter { - // Only use JitPack for the `gson-record-type-adapter-factory` library - includeModule("com.github.Marcono1234", "gson-record-type-adapter-factory") - } - } -} - -java { - toolchain { - languageVersion.set(JavaLanguageVersion.of(17)) - } -} -dependencies { - implementation project(':code:common:service') - implementation project(':code:common:service-discovery') - implementation project(':code:common:service-client') - - implementation 'org.jetbrains:annotations:24.0.0' - - implementation 'org.projectlombok:lombok:1.18.24' - annotationProcessor 'org.projectlombok:lombok:1.18.24' - implementation 'com.github.jknack:handlebars:4.3.1' - implementation 'com.github.jknack:handlebars-markdown:4.2.1' - - implementation group: 'com.google.code.gson', name: 'gson', version: '2.9.0' - implementation 'io.reactivex.rxjava3:rxjava:3.1.5' - implementation "com.sparkjava:spark-core:2.9.3" - - implementation group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.17.2' - implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.17.2' - implementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.17.2' - - implementation 'org.slf4j:slf4j-api:1.7.36' - testImplementation 'org.slf4j:slf4j-jdk14:2.0.3' - - implementation 'com.google.guava:guava:31.1-jre' - implementation 'com.google.inject:guice:5.1.0' - implementation 'com.github.jnr:jnr-ffi:2.2.12' - implementation 'org.apache.httpcomponents:httpcore:4.4.15' - implementation 'org.apache.httpcomponents:httpclient:4.5.13' - - implementation 'org.jsoup:jsoup:1.15.3' - implementation group: 'com.github.crawler-commons', name: 'crawler-commons', version: '1.2' - - implementation 'org.mariadb.jdbc:mariadb-java-client:3.0.6' - implementation group: 'net.sf.trove4j', name: 'trove4j', version: '3.0.3' - - implementation 'com.zaxxer:HikariCP:5.0.1' - - implementation 'org.apache.opennlp:opennlp-tools:1.9.4' - implementation 'io.prometheus:simpleclient:0.16.0' - implementation 'io.prometheus:simpleclient_servlet:0.16.0' - implementation 'io.prometheus:simpleclient_httpserver:0.16.0' - implementation 'io.prometheus:simpleclient_hotspot:0.16.0' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.3' - - implementation group: 'org.yaml', name: 'snakeyaml', version: '1.30' - - implementation 'commons-net:commons-net:3.8.0' - implementation 'org.eclipse.jgit:org.eclipse.jgit:5.12.0.202106070339-r' - implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.jsch:5.12.0.202106070339-r' - - implementation group: 'org.apache.commons', name: 'commons-compress', version: '1.21' - implementation 'edu.stanford.nlp:stanford-corenlp:4.4.0' - - implementation group: 'it.unimi.dsi', name: 'fastutil', version: '8.5.8' - implementation 'org.roaringbitmap:RoaringBitmap:0.9.32' - - implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.29' - implementation 'com.github.Marcono1234:gson-record-type-adapter-factory:0.2.0' - - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2' - testImplementation 'org.mockito:mockito-junit-jupiter:4.5.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' - testCompileOnly 'org.projectlombok:lombok:1.18.24' - testImplementation 'org.projectlombok:lombok:1.18.24' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.24' - - testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.5.1' - - implementation 'net.agkn:hll:1.6.0' - -} - -test { - useJUnitPlatform() -} - -task fastTests(type: Test) { - useJUnitPlatform { - excludeTags "slow" - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/WmsaHome.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/WmsaHome.java deleted file mode 100644 index acf53e8f..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/WmsaHome.java +++ /dev/null @@ -1,56 +0,0 @@ -package nu.marginalia.wmsa; - - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; -import java.util.Properties; - -public class WmsaHome { - private static final String DEFAULT = "/var/lib/wmsa"; - - public static Path getHomePath() { - var retStr = Optional.ofNullable(System.getenv("WMSA_HOME")).orElse(DEFAULT); - - var ret = Path.of(retStr); - if (!Files.isDirectory(ret)) { - throw new IllegalStateException("Could not find WMSA_HOME, either set environment variable or ensure " + DEFAULT + " exists"); - } - return ret; - } - - public static Path getDisk(String name) { - var pathStr = getDiskProperties().getProperty(name); - if (null == pathStr) { - throw new RuntimeException("Disk " + name + " was not configured"); - } - Path p = Path.of(pathStr); - if (!Files.isDirectory(p)) { - throw new RuntimeException("Disk " + name + " does not exist or is not a directory!"); - } - return p; - } - - public static Properties getDiskProperties() { - Path settingsFile = getHomePath().resolve("conf/disks.properties"); - - if (!Files.isRegularFile(settingsFile)) { - throw new RuntimeException("Could not find disk settings " + settingsFile); - } - - try (var is = Files.newInputStream(settingsFile)) { - var props = new Properties(); - props.load(is); - return props; - } - catch (IOException ex) { - throw new RuntimeException(ex); - } - } - - private static final boolean debugMode = Boolean.getBoolean("wmsa-debug"); - public static boolean isDebug() { - return debugMode; - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastFetcher.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastFetcher.java deleted file mode 100644 index 58fad4c9..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastFetcher.java +++ /dev/null @@ -1,112 +0,0 @@ -package nu.marginalia.wmsa.podcasts; - -import com.google.common.escape.Escaper; -import com.google.common.net.PercentEscaper; -import lombok.Getter; -import nu.marginalia.wmsa.podcasts.model.*; -import org.jetbrains.annotations.NotNull; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Element; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URL; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -@Getter -public class PodcastFetcher { - - private final List allEpisodes = new ArrayList<>(); - private final List allPodcasts = new ArrayList<>(); - private final Escaper urlEscaper = new PercentEscaper("", true); - - private final static Logger logger = LoggerFactory.getLogger(PodcastFetcher.class); - - private final DateTimeFormatter readableIsoDate = - (new DateTimeFormatterBuilder()).parseCaseInsensitive().append(DateTimeFormatter.ISO_LOCAL_DATE).appendLiteral(' ').append( - DateTimeFormatter.ISO_LOCAL_TIME).toFormatter(); - - public Optional fetchPodcast(String name, String url) { - try { - logger.info("Fetching podcast {} : {}", name, url); - - var doc = Jsoup.parse(new URL(url), 10_000); - - String title = doc.selectFirst("channel > title").text(); - String description = doc.selectFirst("channel > description").text(); - String link = doc.selectFirst("channel > link").text(); - - var podcast = new Podcast(new PodcastMetadata(title, description, name, link)); - doc.getElementsByTag("item").forEach(item -> { - try { - PodcastEpisode episode = fetchEpisode(name, title, item); - podcast.episodes.add(episode); - allEpisodes.add(episode); - } - catch (Exception ex) { - logger.error("Failed to fetch podcast episode", ex); - } - }); - - allPodcasts.add(podcast); - return Optional.of(podcast); - } catch (IOException e) { - logger.error("Failed to fetch podcast", e); - return Optional.empty(); - } - - } - - @NotNull - private PodcastEpisode fetchEpisode(String name, String title, org.jsoup.nodes.Element item) { - String epTitle = item.getElementsByTag("title").text(); - String epGuid = name+":"+escapeUrlString(item.getElementsByTag("guid").text()); - String epDescription = item.getElementsByTag("description").text(); - String epPubDate = getPubDate(item); - String epUrl = item.getElementsByTag("enclosure").attr("url"); - - return new PodcastEpisode(name, title, epGuid, epTitle, epDescription, epPubDate, epUrl); - } - - @NotNull - private String getPubDate(Element item) { - try { - return ZonedDateTime.parse(item.getElementsByTag("pubDate").text(), - DateTimeFormatter.RFC_1123_DATE_TIME) - .format(readableIsoDate); - } - catch (Exception ex) { - logger.error("Failed to parse date", ex); - return item.getElementsByTag("pubDate").text(); - } - } - - private String escapeUrlString(String s) { - return urlEscaper.escape(s).replace("%", "_"); - } - - public PodcastNewEpisodes getNewEpisodes() { - return new PodcastNewEpisodes(allEpisodes - .stream() - .sorted(Comparator.comparing(PodcastEpisode::getDateUploaded).reversed()).limit(10) - .collect(Collectors.toList())); - } - - public PodcastListing getListing() { - final var metadatas = allPodcasts.stream() - .map(Podcast::getMetadata) - .sorted(Comparator.comparing(PodcastMetadata::getTitle)) - .collect(Collectors.toList()); - - return - new PodcastListing(metadatas); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperMain.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperMain.java deleted file mode 100644 index 8e006d96..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperMain.java +++ /dev/null @@ -1,31 +0,0 @@ -package nu.marginalia.wmsa.podcasts; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.service.MainClass; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.service.module.ConfigurationModule; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.wmsa.renderer.WmsaServiceDescriptors; - -public class PodcastScraperMain extends MainClass { - - private final PodcastScraperService service; - - @Inject - public PodcastScraperMain(PodcastScraperService service) { - this.service = service; - } - - public static void main(String... args) { - - init(ServiceId.Other_PodcastScraper, args); - - Injector injector = Guice.createInjector( - new ConfigurationModule(WmsaServiceDescriptors.descriptors, ServiceId.Other_PodcastScraper)); - injector.getInstance(PodcastScraperMain.class); - injector.getInstance(Initialization.class).setReady(); - } - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperService.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperService.java deleted file mode 100644 index 8c60c8d8..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/PodcastScraperService.java +++ /dev/null @@ -1,78 +0,0 @@ -package nu.marginalia.wmsa.podcasts; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.client.Context; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.MetricsServer; -import nu.marginalia.service.server.Service; -import nu.marginalia.wmsa.podcasts.model.Podcast; -import nu.marginalia.wmsa.podcasts.model.PodcastEpisode; -import nu.marginalia.wmsa.renderer.client.RendererClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Spark; - -import java.util.Map; -import java.util.concurrent.TimeUnit; - -public class PodcastScraperService extends Service { - - private final Map podcastUrls = Map.of( - "SBS", "https://feeds.simplecast.com/Fxu1mrhe", - "hopwag", "https://feed.podbean.com/hopwag/feed.xml", - "philosophizethis", "https://philosophizethis.libsyn.com/rss", - "PEL", "https://partiallyexaminedlife.libsyn.com/rss", - "IOT", "https://podcasts.files.bbci.co.uk/b006qykl.rss", - "SaturaLanx", "https://anchor.fm/s/2c536214/podcast/rss", - "ControversiesInChurchHistory", "https://anchor.fm/s/9b43760/podcast/rss", - "readmeapoem", "https://rss.acast.com/readmeapoem", - "HoL", "https://feeds.megaphone.fm/history-of-literature", - "Revolutions", "https://revolutionspodcast.libsyn.com/rss" - ); - - private final RendererClient rendererClient; - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Initialization initialization; - - @Inject - public PodcastScraperService(@Named("service-host") String ip, - @Named("service-port") Integer port, - RendererClient rendererClient, - Initialization initialization, - MetricsServer metricsServer) { - super(ip, port, initialization, metricsServer); - this.rendererClient = rendererClient; - this.initialization = initialization; - - Spark.awaitInitialization(); - - Schedulers.io().schedulePeriodicallyDirect(this::fetchPods, 0, 1, TimeUnit.HOURS); - } - - private void fetchPods() { - try { - PodcastFetcher fetcher = new PodcastFetcher(); - - podcastUrls.forEach(fetcher::fetchPodcast); - - rendererClient.render(Context.internal("podcast"), fetcher.getNewEpisodes()).blockingSubscribe(); - rendererClient.render(Context.internal("podcast"), fetcher.getListing()).blockingSubscribe(); - - for (Podcast podcast : fetcher.getAllPodcasts()) { - rendererClient.render(Context.internal("podcast"), podcast).blockingSubscribe(); - } - for (PodcastEpisode episode : fetcher.getAllEpisodes()) { - rendererClient.render(Context.internal("podcast"), episode).blockingSubscribe(); - } - } - catch (RuntimeException ex) { - logger.error("Uncaught exception", ex); - } - } - - public void start() { - logger.info("Started"); - } -} \ No newline at end of file diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/Podcast.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/Podcast.java deleted file mode 100644 index 7c3ae0ae..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/Podcast.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.wmsa.podcasts.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.ArrayList; -import java.util.List; - -@AllArgsConstructor @Getter @Setter @ToString -public class Podcast { - public final PodcastMetadata metadata; - - public final List episodes = new ArrayList<>(); -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastEpisode.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastEpisode.java deleted file mode 100644 index 255de6a0..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastEpisode.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.wmsa.podcasts.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.ToString; - -@AllArgsConstructor @Getter @ToString -public class PodcastEpisode { - public final String podcastId; - public final String podcastName; - public final String guid; - public final String title; - public final String description; - public final String dateUploaded; - public final String mp3url; -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastListing.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastListing.java deleted file mode 100644 index 3a3e917a..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastListing.java +++ /dev/null @@ -1,16 +0,0 @@ -package nu.marginalia.wmsa.podcasts.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -@AllArgsConstructor -@Getter -@Setter -@ToString -public class PodcastListing { - public final List podcasts; -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastMetadata.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastMetadata.java deleted file mode 100644 index 8f6aad6c..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastMetadata.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.wmsa.podcasts.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -@AllArgsConstructor -@Getter -@Setter -@ToString -public class PodcastMetadata { - public final String title; - public final String description; - public final String id; - public final String extLink; -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastNewEpisodes.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastNewEpisodes.java deleted file mode 100644 index c7562e60..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/podcasts/model/PodcastNewEpisodes.java +++ /dev/null @@ -1,11 +0,0 @@ -package nu.marginalia.wmsa.podcasts.model; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.List; - -@AllArgsConstructor @Getter -public class PodcastNewEpisodes { - public final List episodes; -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/PodcastRendererService.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/PodcastRendererService.java deleted file mode 100644 index 3398a2ff..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/PodcastRendererService.java +++ /dev/null @@ -1,128 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import com.google.gson.Gson; -import com.google.inject.Inject; -import lombok.SneakyThrows; -import nu.marginalia.client.Context; -import nu.marginalia.wmsa.podcasts.model.Podcast; -import nu.marginalia.wmsa.podcasts.model.PodcastListing; -import nu.marginalia.wmsa.podcasts.model.PodcastNewEpisodes; -import nu.marginalia.wmsa.podcasts.model.PodcastEpisode; -import nu.marginalia.wmsa.renderer.mustache.MustacheRenderer; -import nu.marginalia.wmsa.renderer.mustache.RendererFactory; -import nu.marginalia.wmsa.resource_store.ResourceStoreClient; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; -import spark.Spark; - -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.concurrent.TimeUnit; - -public class PodcastRendererService { - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Gson gson = new Gson(); - - private final RendererFactory rendererFactory = new RendererFactory(); - - private final MustacheRenderer newsRenderer; - private final MustacheRenderer episodeRenderer; - private final MustacheRenderer listingRenderer; - private final MustacheRenderer podcastRenderer; - - private final ResourceStoreClient resourceStoreClient; - - - @Inject @SneakyThrows - public PodcastRendererService(ResourceStoreClient resourceStoreClient) { - this.resourceStoreClient = resourceStoreClient; - newsRenderer = rendererFactory.renderer( "podcast/new"); - episodeRenderer = rendererFactory.renderer( "podcast/episode"); - listingRenderer = rendererFactory.renderer( "podcast/listing"); - podcastRenderer = rendererFactory.renderer( "podcast/podcast"); - } - - public void start() { - Spark.post("/render/podcast", this::renderPodcast); - Spark.post("/render/podcast/episode", this::renderPodcastEpisode); - Spark.post("/render/podcast/new", this::renderPodcastNew); - Spark.post("/render/podcast/listing", this::renderPodcastListing); - } - - private Object renderPodcastListing(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, PodcastListing.class); - - logger.info("renderPodcastListing()"); - - var resource = new RenderedResource("list.html", - getRetentionTime(), - listingRenderer.render(req)); - - storeResource(request, resource); - - return ""; - } - - - private Object renderPodcast(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, Podcast.class); - - logger.info("renderPodcast({})", req.metadata.id); - - var resource = new RenderedResource(req.metadata.id+".html", - getRetentionTime(), - podcastRenderer.render(req)); - - storeResource(request, resource); - - return ""; - } - - private Object renderPodcastEpisode(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, PodcastEpisode.class); - Context.fromRequest(request); - - logger.info("renderPodcastEpisode({}/{})", req.podcastName, req.guid); - var resource = new RenderedResource(req.guid+".html", - getRetentionTime(), - episodeRenderer.render(req)); - - storeResource(request, resource); - - return ""; - } - - private Object renderPodcastNew(Request request, Response response) { - var requestText = request.body(); - var req = gson.fromJson(requestText, PodcastNewEpisodes.class); - - logger.info("renderPodcastNew()"); - - var resource = new RenderedResource("new.html", - getRetentionTime(), - newsRenderer.render(req)); - - storeResource(request, resource); - - return ""; - } - - - private LocalDateTime getRetentionTime() { - return LocalDateTime.now().plus(24, ChronoUnit.HOURS); - } - - private void storeResource(Request request, RenderedResource resource) { - resourceStoreClient.putResource(Context.fromRequest(request), "podcast", resource) - .timeout(10, TimeUnit.SECONDS) - .blockingSubscribe(); - } - - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererMain.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererMain.java deleted file mode 100644 index 98d7fd83..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererMain.java +++ /dev/null @@ -1,29 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.service.MainClass; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.service.module.ConfigurationModule; -import nu.marginalia.service.server.Initialization; - -public class RendererMain extends MainClass { - private final RendererService service; - - @Inject - public RendererMain(RendererService service - ) { - this.service = service; - } - - public static void main(String... args) { - init(ServiceId.Other_Renderer, args); - - Injector injector = Guice.createInjector( - new RendererModule(), - new ConfigurationModule(WmsaServiceDescriptors.descriptors, ServiceId.Other_Renderer)); - injector.getInstance(RendererMain.class); - injector.getInstance(Initialization.class).setReady(); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererModule.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererModule.java deleted file mode 100644 index 99422709..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererModule.java +++ /dev/null @@ -1,8 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import com.google.inject.AbstractModule; - -public class RendererModule extends AbstractModule { - public void configure() { - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java deleted file mode 100644 index 86dfcc9a..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/RendererService.java +++ /dev/null @@ -1,35 +0,0 @@ -package nu.marginalia.wmsa.renderer; - - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.MetricsServer; -import nu.marginalia.service.server.Service; -import nu.marginalia.wmsa.resource_store.ResourceStoreClient; - - -public class RendererService extends Service { - private final ResourceStoreClient resourceStoreClient; - - - @Inject - public RendererService(ResourceStoreClient resourceStoreClient, - @Named("service-host") String ip, - @Named("service-port") Integer port, - PodcastRendererService podcastRendererService, - Initialization initialization, - MetricsServer metricsServer - ) { - super(ip, port, initialization, metricsServer); - - this.resourceStoreClient = resourceStoreClient; - - podcastRendererService.start(); - } - - public boolean isReady() { - return resourceStoreClient.isAccepting(); - } - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/ServerStatusModel.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/ServerStatusModel.java deleted file mode 100644 index fea3ee41..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/ServerStatusModel.java +++ /dev/null @@ -1,10 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -@AllArgsConstructor @Getter -public class ServerStatusModel { - public final String server; - public final String status; -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/WmsaServiceDescriptors.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/WmsaServiceDescriptors.java deleted file mode 100644 index ca25440f..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/WmsaServiceDescriptors.java +++ /dev/null @@ -1,17 +0,0 @@ -package nu.marginalia.wmsa.renderer; - -import nu.marginalia.service.descriptor.ServiceDescriptor; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; - -import java.util.List; - -public class WmsaServiceDescriptors { - public static ServiceDescriptors descriptors = new ServiceDescriptors( - List.of( - new ServiceDescriptor(ServiceId.Other_ResourceStore, 5000), - new ServiceDescriptor(ServiceId.Other_Renderer, 5002), - new ServiceDescriptor(ServiceId.Other_PodcastScraper, 5013))); - - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java deleted file mode 100644 index 809a7cfa..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/client/RendererClient.java +++ /dev/null @@ -1,53 +0,0 @@ -package nu.marginalia.wmsa.renderer.client; - -import com.google.gson.Gson; -import io.reactivex.rxjava3.core.Observable; -import lombok.SneakyThrows; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; -import nu.marginalia.client.HttpStatusCode; -import nu.marginalia.service.descriptor.HostsFile; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.wmsa.podcasts.model.Podcast; -import nu.marginalia.wmsa.podcasts.model.PodcastEpisode; -import nu.marginalia.wmsa.podcasts.model.PodcastListing; -import nu.marginalia.wmsa.podcasts.model.PodcastNewEpisodes; -import nu.marginalia.client.exception.TimeoutException; - -import javax.inject.Inject; -import java.util.concurrent.TimeUnit; - - -public class RendererClient extends AbstractDynamicClient { - @Inject - public RendererClient(ServiceDescriptors descriptors) { - super(descriptors.forId(ServiceId.Index), new HostsFile(), Gson::new); - } - - @SneakyThrows - public Observable render(Context ctx, PodcastNewEpisodes req) { - return post(ctx, "/render/podcast/new", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderPodcastNew()"))); - } - - - @SneakyThrows - public Observable render(Context ctx, PodcastEpisode req) { - return post(ctx, "/render/podcast/episode", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderPodcastEpisode()"))); - } - - @SneakyThrows - public Observable render(Context ctx, PodcastListing req) { - return post(ctx, "/render/podcast/listing", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderPodcastListing()"))); - } - - - @SneakyThrows - public Observable render(Context ctx, Podcast req) { - return post(ctx, "/render/podcast", req) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("RendererClient.renderPodcastEpisode()"))); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/MustacheRenderer.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/MustacheRenderer.java deleted file mode 100644 index 90e50754..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/MustacheRenderer.java +++ /dev/null @@ -1,61 +0,0 @@ -package nu.marginalia.wmsa.renderer.mustache; - -import com.github.jknack.handlebars.*; -import com.github.jknack.handlebars.helper.ConditionalHelpers; -import com.github.jknack.handlebars.io.ClassPathTemplateLoader; -import com.github.jknack.handlebars.io.TemplateLoader; -import lombok.SneakyThrows; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.List; -import java.util.Map; - -public class MustacheRenderer { - Template template; - private final Logger logger = LoggerFactory.getLogger(getClass()); - - MustacheRenderer(String templateFile) throws IOException { - - TemplateLoader loader = new ClassPathTemplateLoader(); - loader.setPrefix("/templates"); - loader.setSuffix(".hdb"); - - var handlebars = new Handlebars(loader); - handlebars.registerHelpers(ConditionalHelpers.class); - handlebars.registerHelper("md", new MarkdownHelper()); - - try { - template = handlebars.compile(templateFile); - } - catch (FileNotFoundException ex) { - logger.error("Kunde inte ladda template " + templateFile, ex); - System.exit(2); - } - catch (HandlebarsException ex) { - logger.error("Kunde inte instantiera mall " + templateFile, ex); - System.exit(2); - } - } - - @SneakyThrows - public String render(T model) { - return template.apply(model); - } - - @SneakyThrows - public String render(T model, String name, List children) { - Context ctx = Context.newBuilder(model).combine(name, children).build(); - - return template.apply(ctx); - } - - @SneakyThrows - public String render(T model, Map children) { - Context ctx = Context.newBuilder(model).combine(children).build(); - return template.apply(ctx); - } - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/RendererFactory.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/RendererFactory.java deleted file mode 100644 index 0fe81abf..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/renderer/mustache/RendererFactory.java +++ /dev/null @@ -1,13 +0,0 @@ -package nu.marginalia.wmsa.renderer.mustache; - -import java.io.IOException; - -public class RendererFactory { - - public RendererFactory() { - } - - public MustacheRenderer renderer(String template) throws IOException { - return new MustacheRenderer<>(template); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceEntityStore.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceEntityStore.java deleted file mode 100644 index ebfda218..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceEntityStore.java +++ /dev/null @@ -1,247 +0,0 @@ -package nu.marginalia.wmsa.resource_store; - -import com.google.gson.Gson; -import com.google.inject.name.Named; -import io.prometheus.client.Counter; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import javax.inject.Inject; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Base64; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - - -public class ResourceEntityStore { - private final Map resources = new HashMap<>(); - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final Path dataPath; - private final Gson gson = new Gson(); - private final Base64.Encoder b64encoder = Base64.getEncoder(); - - private final static Counter wmsa_resource_store_count - = Counter.build("wmsa_resource_store_count", "number of items in the resource store") - .register(); - private final static Counter wmsa_resource_store_eviction_count - = Counter.build("wmsa_resource_store_eviction_count", "evicted items") - .register(); - - @Inject - public ResourceEntityStore(@Named("data-path") Path dataPath) { - this.dataPath = dataPath; - - Schedulers.io().scheduleDirect(() -> loadResourcesFromDisk(dataPath)); - Schedulers.io().schedulePeriodicallyDirect(() -> purgeFileSystem(dataPath), 1, 1, TimeUnit.HOURS); - } - - public ResourceEntityStore(@Named("data-path") Path dataPath, boolean immediate) { - this.dataPath = dataPath; - - loadResourcesFromDisk(dataPath); - } - - public ResourceEntityStore() { - this.dataPath = null; - } - - public RenderedResource getResource(String domain, String resource) { - Lock readLock = lock.readLock(); - try { - readLock.lock(); - return resources.get(getKey(domain, resource)); - } - finally { - readLock.unlock(); - } - } - - public void putResource(String domain, String resource, RenderedResource data) { - RenderedResource oldResource = loadResource(domain, resource, data); - - wmsa_resource_store_count.inc(); - if (dataPath != null) { - Path domainPath = dataPath.resolve(domain); - if (!domainPath.toFile().isDirectory()) { - domainPath.toFile().mkdir(); - } - - if (oldResource != null) { - try { - Path oldResourcePath = domainPath.resolve(oldResource.diskFileName()); - oldResourcePath.toFile().delete(); - } - catch (Exception ex) { - logger.error("Failed to remove old resource {}/{}", domain, oldResource.diskFileName()); - } - } - - Path resourcePath = domainPath.resolve(data.diskFileName()); - try { - Files.writeString(resourcePath, gson.toJson(data)); - } catch (IOException e) { - logger.error("Failed to write resource {}/{}", domain, resource); - logger.error("Exception", e); - } - - - } - } - - @Nullable - private RenderedResource loadResource(String domain, String resource, RenderedResource data) { - Lock writeLock = lock.writeLock(); - RenderedResource oldResource; - try { - writeLock.lock(); - oldResource = resources.put(getKey(domain, resource), data); - } - finally { - writeLock.unlock(); - } - return oldResource; - } - - private String getKey(String domain, String resource) { - return domain + "/" + resource; - } - - - public void reapStaleResources() { - Lock writeLock = lock.writeLock(); - try { - writeLock.lock(); - List expiredResources = resources.entrySet().stream() - .filter(entry -> entry.getValue().isExpired()) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - - for (String resource : expiredResources) { - logger.info("Reaping expired resource \"{}\"", resource); - var res = resources.remove(resource); - wmsa_resource_store_eviction_count.inc(); - - if (dataPath != null) { - File resourceFile = dataPath.resolve(res.diskFileName()).toFile(); - if (resourceFile.exists()) { - resourceFile.delete(); - } - } - } - } - finally { - writeLock.unlock(); - } - } - - public int numResources() { - Lock readLock = lock.readLock(); - try { - readLock.lock(); - return resources.size(); - } - finally { - readLock.unlock(); - } - } - - public long resourceSize() { - Lock readLock = lock.readLock(); - try { - readLock.lock(); - return resources.values().stream().mapToLong(RenderedResource::size).sum(); - } - finally { - readLock.unlock(); - } - } - - public void loadResourcesFromDisk(Path dataPath) { - File dataDir = dataPath.toFile(); - - for (var dir : dataDir.listFiles()) { - if (!dir.isDirectory()) { - logger.warn("Junk file {} in data directory", dir); - } - else { - for (var file : dir.listFiles()) { - try { - loadFromFile(dir.getName(), file); - } - catch (Exception ex) { - logger.error("Failed to load file {}", file); - logger.error("Failed to load resource from disk", ex); - } - } - } - } - } - - public void purgeFileSystem(Path dataPath) { - File dataDir = dataPath.toFile(); - - for (var dir : dataDir.listFiles()) { - if (!dir.isDirectory()) { - logger.warn("Junk file {} in data directory", dir); - } - else { - for (var file : dir.listFiles()) { - try { - purgeFile(file); - } - catch (Exception ex) { - logger.error("Failed to purge resource from disk", ex); - } - } - } - } - } - - private void purgeFile(File file) throws IOException { - String json = Files.readString(file.toPath(), Charset.defaultCharset()); - var resource = gson.fromJson(json, RenderedResource.class); - - if (resource.isExpired()) { - logger.info("Deleting expired resource {}", file); - - file.delete(); - } - - } - - - private void loadFromFile(String domain, File file) { - try { - String json = Files.readString(file.toPath(), Charset.defaultCharset()); - var resource = gson.fromJson(json, RenderedResource.class); - - if (resource.isExpired() || resources.containsKey(getKey(domain, resource.getFilename()))) { - logger.info("Deleting expired resource {}", file); - - file.delete(); - } - else { - logger.info("Re-loading resource {}", file); - loadResource(domain, resource.getFilename(), resource); - } - } catch (IOException e) { - logger.error("Could not read file {}", file); - } - } - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreClient.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreClient.java deleted file mode 100644 index d9333638..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreClient.java +++ /dev/null @@ -1,49 +0,0 @@ -package nu.marginalia.wmsa.resource_store; - -import com.google.gson.Gson; -import io.reactivex.rxjava3.core.Observable; -import io.reactivex.rxjava3.schedulers.Schedulers; -import nu.marginalia.client.AbstractDynamicClient; -import nu.marginalia.client.Context; -import nu.marginalia.client.HttpStatusCode; -import nu.marginalia.service.descriptor.HostsFile; -import nu.marginalia.service.descriptor.ServiceDescriptors; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import nu.marginalia.client.exception.TimeoutException; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.time.LocalDateTime; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -@Singleton -public class ResourceStoreClient extends AbstractDynamicClient { - - @Inject - public ResourceStoreClient(ServiceDescriptors descriptors) { - super(descriptors.forId(ServiceId.Other_ResourceStore), new HostsFile(), Gson::new); - } - - public Observable getResource(Context ctx, String domain, String resource) { - return get(ctx, "/"+domain+"/"+resource) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("ResourceStoreClient.getResource()"))) - ; - } - - public Observable putResource(Context ctx, String domain, RenderedResource data) { - return post(ctx, "/"+domain, data) - .timeout(5, TimeUnit.SECONDS, Observable.error(new TimeoutException("ResourceStoreClient.putResource()"))); - - } - - public Observable cacheResource(Context ctx, String domain, String resource, Supplier generator, LocalDateTime expiry) { - return getResource(ctx, domain, resource) - .onErrorReturn(e -> { - var renderedResource = new RenderedResource(resource, expiry, generator.get()); - putResource(ctx, "wiki", renderedResource).subscribeOn(Schedulers.io()).blockingSubscribe(); - return renderedResource.data; - }); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreMain.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreMain.java deleted file mode 100644 index f3e84dee..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreMain.java +++ /dev/null @@ -1,30 +0,0 @@ -package nu.marginalia.wmsa.resource_store; - -import com.google.inject.Guice; -import com.google.inject.Inject; -import com.google.inject.Injector; -import nu.marginalia.service.MainClass; -import nu.marginalia.service.id.ServiceId; -import nu.marginalia.service.module.ConfigurationModule; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.wmsa.renderer.WmsaServiceDescriptors; - -public class ResourceStoreMain extends MainClass { - private final ResourceStoreService service; - - @Inject - public ResourceStoreMain(ResourceStoreService service) { - this.service = service; - } - - public static void main(String... args) { - init(ServiceId.Other_ResourceStore, args); - - Injector injector = Guice.createInjector( - new ResourceStoreModule(), - new ConfigurationModule(WmsaServiceDescriptors.descriptors, ServiceId.Other_ResourceStore) - ); - injector.getInstance(ResourceStoreMain.class); - injector.getInstance(Initialization.class).setReady(); - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java deleted file mode 100644 index 70db9271..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreModule.java +++ /dev/null @@ -1,15 +0,0 @@ -package nu.marginalia.wmsa.resource_store; - -import com.google.inject.AbstractModule; -import com.google.inject.name.Names; -import nu.marginalia.wmsa.WmsaHome; - -import java.nio.file.Path; - -public class ResourceStoreModule extends AbstractModule { - public void configure() { - bind(Path.class).annotatedWith(Names.named("data-path")).toInstance(WmsaHome.getDisk("resource-store")); - } - - -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreService.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreService.java deleted file mode 100644 index 45415275..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/ResourceStoreService.java +++ /dev/null @@ -1,158 +0,0 @@ -package nu.marginalia.wmsa.resource_store; - -import com.google.gson.Gson; -import com.google.inject.Inject; -import com.google.inject.name.Named; -import io.reactivex.rxjava3.schedulers.Schedulers; -import lombok.SneakyThrows; -import nu.marginalia.client.Context; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.MetricsServer; -import nu.marginalia.service.server.Service; -import nu.marginalia.service.server.StaticResources; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import org.apache.http.HttpStatus; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Request; -import spark.Response; -import spark.Spark; -import spark.resource.ClassPathResource; -import spark.staticfiles.MimeType; - -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.concurrent.TimeUnit; - -public class ResourceStoreService extends Service { - private final Gson gson = new Gson(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final long startTime = LocalDateTime.now().toEpochSecond(ZoneOffset.UTC); - - private final ResourceEntityStore resourceStore; - private StaticResources staticResources; - - @Inject - public ResourceStoreService(@Named("service-host") String ip, - @Named("service-port") Integer port, - ResourceEntityStore resourceStore, - Initialization initialization, - MetricsServer metricsServer, - StaticResources staticResources - ) { - super(ip, port, initialization, metricsServer); - this.resourceStore = resourceStore; - this.staticResources = staticResources; - - Schedulers.io().schedulePeriodicallyDirect(resourceStore::reapStaleResources, - 5, 5, TimeUnit.MINUTES); - - Spark.get("/public/*", this::getDefaultResource); - - Spark.get("/:domain/*", this::getResource); - Spark.post("/:domain", this::storeResource); - - } - - private Object getDefaultResource(Request request, Response response) { - String headerDomain = request.headers("X-Domain"); - - if (headerDomain == null) { - Spark.halt(404); - } - - var splat = request.splat(); - var resource = splat.length == 0 ? "index.html" : splat[0]; - - return getResource(request, response, headerDomain, resource); - - } - - private Object storeResource(Request request, Response response) { - var domain = request.params("domain"); - var data = gson.fromJson(request.body(), RenderedResource.class); - - logger.info("storeResource({}/{}, {})", domain, data.filename, data.etag()); - - resourceStore.putResource(domain, data.filename, data); - - Spark.halt(HttpStatus.SC_ACCEPTED); - return null; - } - - private Object getResource(Request request, Response response) { - String headerDomain = request.headers("X-Domain"); - var domain = request.params("domain"); - - if (headerDomain != null && !domain.equals(headerDomain)) { - logger.warn("{} - domain mismatch: Header = {}, request = {}", Context.fromRequest(request), headerDomain, domain); - Spark.halt(403); - } - - var splat = request.splat(); - var resource = splat.length == 0 ? "index.html" : splat[0]; - - return getResource(request, response, domain, resource); - } - - private String getResource(Request request, Response response, String domain, String resource) { - - var data = resourceStore.getResource(domain, resource); - - if (data != null) { - logger.info("getResource({}/{}, {})", domain, resource, data.etag()); - - return serveDynamic(data, request, response); - } - else { - logger.info("getResource({}/{}, static)", domain, resource); - staticResources.serveStatic(domain, resource, request, response); - } - return ""; - } - - private String serveDynamic(RenderedResource data, Request request, Response response) { - handleEtag(data, request, response); - - return data.data; - } - - - - @SneakyThrows - private void handleEtag(RenderedResource page, Request req, Response rsp) { - rsp.header("Cache-Control", "private, must-revalidate"); - - if (!page.filename.endsWith(".txt")) { - rsp.type("text/html"); - } - else { - rsp.type(MimeType.fromResource(new ClassPathResource(page.filename))); - } - final String etag = page.etag(); - - if (etag.equals(req.headers("If-None-Match"))) { - Spark.halt(304); - } - - rsp.header("ETag", etag); - } - - @SneakyThrows - private void handleEtagStatic(ClassPathResource resource, Request req, Response rsp) { - rsp.header("Cache-Control", "public,max-age=3600"); - rsp.type(MimeType.fromResource(resource)); - - final String etag = staticResourceEtag(resource.getFilename()); - - if (etag.equals(req.headers("If-None-Match"))) { - Spark.halt(304); - } - - rsp.header("ETag", etag); - } - - private String staticResourceEtag(String resource) { - return "\"" + resource.hashCode() + "-" + startTime + "\""; - } -} diff --git a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/model/RenderedResource.java b/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/model/RenderedResource.java deleted file mode 100644 index f445a6c7..00000000 --- a/other/wmsa_old/src/main/java/nu/marginalia/wmsa/resource_store/model/RenderedResource.java +++ /dev/null @@ -1,46 +0,0 @@ -package nu.marginalia.wmsa.resource_store.model; - -import lombok.Getter; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; - -@Getter -public class RenderedResource { - public final String filename; - public final String data; - public final String genTime = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - public final long genTimeMillis = System.currentTimeMillis(); - public final String expiry; - public final boolean requireLogin; - - public RenderedResource(String filename, LocalDateTime expiryDate, String data) { - this.filename = filename; - this.data = data; - this.expiry = expiryDate.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - this.requireLogin = false; - } - public RenderedResource(String filename, LocalDateTime expiryDate, String data, boolean requireLogin) { - this.filename = filename; - this.data = data; - this.expiry = expiryDate.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME); - this.requireLogin = requireLogin; - } - public boolean isExpired() { - var expiryDate = LocalDateTime.parse(expiry, DateTimeFormatter.ISO_LOCAL_DATE_TIME); - return expiryDate.isBefore(LocalDateTime.now()); - - } - - public String etag() { - return "\"" + genTime.hashCode() + "-" + data.hashCode() + "\""; - } - - public String diskFileName() { - return filename.hashCode() + "-" + data.hashCode() + ".html"; - } - - public long size() { - return 2L*(data.length()+filename.length()); - } -} diff --git a/other/wmsa_old/src/main/resources/static/podcast/style.css b/other/wmsa_old/src/main/resources/static/podcast/style.css deleted file mode 100644 index 1939da58..00000000 --- a/other/wmsa_old/src/main/resources/static/podcast/style.css +++ /dev/null @@ -1,3 +0,0 @@ -.headline:visited { - color: #5f3945 !important; -} \ No newline at end of file diff --git a/other/wmsa_old/src/main/resources/templates/podcast/episode.hdb b/other/wmsa_old/src/main/resources/templates/podcast/episode.hdb deleted file mode 100644 index dc1e4306..00000000 --- a/other/wmsa_old/src/main/resources/templates/podcast/episode.hdb +++ /dev/null @@ -1,35 +0,0 @@ - - - - {{podcastName}}: {{title}} - - - - - - -
    - -
    - - - - diff --git a/other/wmsa_old/src/main/resources/templates/podcast/listing.hdb b/other/wmsa_old/src/main/resources/templates/podcast/listing.hdb deleted file mode 100644 index 0c076762..00000000 --- a/other/wmsa_old/src/main/resources/templates/podcast/listing.hdb +++ /dev/null @@ -1,33 +0,0 @@ - - - - Podcasts: Nya avsnitt - - - - - - -
    - -
    -
    -

    Podcasts

    -
    - {{#each podcasts}} -
    - {{title}} -
    -
    -

    {{{description}}}

    -
    - {{/each}} -
    -
    - - - diff --git a/other/wmsa_old/src/main/resources/templates/podcast/new.hdb b/other/wmsa_old/src/main/resources/templates/podcast/new.hdb deleted file mode 100644 index 62c41782..00000000 --- a/other/wmsa_old/src/main/resources/templates/podcast/new.hdb +++ /dev/null @@ -1,34 +0,0 @@ - - - - Podcasts: Nya avsnitt - - - - - - -
    - -
    -
    -

    Nya avsnitt

    -
    - {{#each episodes}} -
    - {{title}} -
    -
    -

    {{podcastName}}
    {{dateUploaded}}

    - -
    - {{/each}} -
    -
    - - - diff --git a/other/wmsa_old/src/main/resources/templates/podcast/podcast.hdb b/other/wmsa_old/src/main/resources/templates/podcast/podcast.hdb deleted file mode 100644 index 8af807ba..00000000 --- a/other/wmsa_old/src/main/resources/templates/podcast/podcast.hdb +++ /dev/null @@ -1,38 +0,0 @@ - - - - Podcasts: {{title}} - - - - - - -
    - -
    -
    -

    {{metadata.title}}

    -

    - {{{metadata.description}}} -

    -

    {{metadata.extLink}}

    -

    Avsnitt

    -
    - {{#each episodes}} -
    - {{title}} -
    -
    -

    {{dateUploaded}}

    -
    - {{/each}} -
    -
    - - - diff --git a/other/wmsa_old/src/test/java/nu/marginalia/resource_store/ResourceStoreServiceTest.java b/other/wmsa_old/src/test/java/nu/marginalia/resource_store/ResourceStoreServiceTest.java deleted file mode 100644 index b1922bc1..00000000 --- a/other/wmsa_old/src/test/java/nu/marginalia/resource_store/ResourceStoreServiceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -package nu.marginalia.resource_store; - -import lombok.SneakyThrows; -import nu.marginalia.client.Context; -import nu.marginalia.service.server.Initialization; -import nu.marginalia.service.server.StaticResources; -import nu.marginalia.wmsa.renderer.WmsaServiceDescriptors; -import nu.marginalia.wmsa.resource_store.ResourceEntityStore; -import nu.marginalia.wmsa.resource_store.ResourceStoreClient; -import nu.marginalia.wmsa.resource_store.ResourceStoreService; -import nu.marginalia.wmsa.resource_store.model.RenderedResource; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import spark.Spark; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.Random; - -import static org.junit.jupiter.api.Assertions.*; - -class ResourceStoreServiceTest { - static ResourceStoreService service; - static ResourceStoreClient client; - - static final int testPort = new Random().nextInt(4000, 10000); - static ResourceEntityStore resourceStore; - static Path tempDir; - private static final Logger logger = LoggerFactory.getLogger(ResourceStoreServiceTest.class); - - @SneakyThrows - @BeforeAll - public static void setUpClass() { - Spark.port(testPort); - System.setProperty("service-name", "renderer"); - - client = new ResourceStoreClient(WmsaServiceDescriptors.descriptors); - client.setServiceRoute("127.0.0.1", testPort); - tempDir = Files.createTempDirectory("ResourceStoreServiceTest"); - resourceStore = new ResourceEntityStore(tempDir); - service = new ResourceStoreService("127.0.0.1", testPort, - resourceStore, new Initialization(), null, new StaticResources()); - - Spark.awaitInitialization(); - } - - @AfterEach - public void clearTempDir() { - for (File f : tempDir.toFile().listFiles()) { - for (File f2 : f.listFiles()) { - logger.debug("Deleting {} -> {}", f2, f2.delete()); - } - logger.debug("Deleting {} -> {}", f, f.delete()); - } - } - - @AfterAll - public static void tearDownAll() { - tempDir.toFile().delete(); - Spark.awaitStop(); - } - - @Test - public void sunnyDay() { - client.putResource(Context.internal(), "test", new RenderedResource("index.html", LocalDateTime.MAX,"Hello World")).blockingSubscribe(); - assertEquals("Hello World", client.getResource(Context.internal(),"test", "index.html").blockingFirst()); - } - - - @Test - public void loadFromDisk() throws InterruptedException { - client.putResource(Context.internal(), "test", new RenderedResource("index.html", LocalDateTime.MAX,"Hello World")).blockingSubscribe(); - client.putResource(Context.internal(), "test", new RenderedResource("expired.html", LocalDateTime.now().minusDays(14),"Hello World")).blockingSubscribe(); - - var resourceStore2 = new ResourceEntityStore(tempDir, true); - Thread.sleep(1000); - var resource = resourceStore2.getResource("test", "index.html"); - - assertNotNull(resource); - assertEquals("Hello World", resource.data); - - assertNull(resourceStore2.getResource("test", "expired.html")); - } - - @Test - public void testReaper() { - client.putResource(Context.internal(), "test", new RenderedResource("index.html", LocalDateTime.now().minusDays(14),"Hello World")).blockingSubscribe(); - assertEquals("Hello World", client.getResource(Context.internal(),"test", "index.html").blockingFirst()); - - resourceStore.reapStaleResources(); - - var ret = client - .getResource(Context.internal(), "test", "index.html") - .onErrorReturnItem("Error") - .blockingFirst(); - assertEquals("Error", ret); - } - - - @Test - public void update() { - client.putResource(Context.internal(), "test", new RenderedResource("index.html", LocalDateTime.MAX,"Hello World")).blockingSubscribe(); - assertEquals("Hello World", client.getResource(Context.internal(),"test", "index.html").blockingFirst()); - client.putResource(Context.internal(), "test", new RenderedResource("index.html", LocalDateTime.MAX,"Hello World 2")).blockingSubscribe(); - assertEquals("Hello World 2", client.getResource(Context.internal(), "test", "index.html").blockingFirst()); - } - - @Test - public void missing() { - var ret = client - .getResource(Context.internal(), "test", "invalid.html") - .onErrorReturnItem("Error") - .blockingFirst(); - assertEquals("Error", ret); - } - -} \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 2aec2434..3f216b4a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -17,7 +17,6 @@ include 'code:libraries:random-write-funnel' include 'code:libraries:next-prime' include 'code:libraries:braille-block-punch-cards' include 'code:libraries:language-processing' -include 'code:libraries:ngram-bloom-filter' include 'code:libraries:term-frequency-dict' include 'code:features-search:screenshots' @@ -53,8 +52,6 @@ include 'code:common:model' include 'code:common:renderer' include 'code:common:process' -include 'code:processes:crawl-job-extractor-process' - include 'code:processes:converting-process' include 'code:processes:crawling-process' include 'code:processes:loading-process' @@ -63,6 +60,9 @@ include 'code:processes:experimental' include 'code:process-models:converting-model' include 'code:process-models:crawling-model' +include 'code:tools:term-frequency-extractor' +include 'code:tools:crawl-job-extractor' + include 'third-party:porterstemmer' include 'third-party:xz' include 'third-party:symspell' @@ -72,9 +72,6 @@ include 'third-party:openzim' include 'third-party:count-min-sketch' include 'third-party:monkey-patch-opennlp' -include 'other:memex' -include 'other:wmsa_old' - include 'tools:screenshot' dependencyResolutionManagement {