mirror of
https://github.com/MarginaliaSearch/MarginaliaSearch.git
synced 2025-02-24 13:19:02 +00:00
dcb43a3308
The most common error when dealing with Slop columns is that they can fall out of sync with each other if the programmer accidentally does a conditional read and forgets to skip. The second most common error is forgetting to close one of the columns in a reader or writer. To deal with both cases, a new class SlopTable is added that keeps track of the lifecycle of all slop columns and performs a check when closing them that they are in sync.
104 lines
3.3 KiB
Groovy
104 lines
3.3 KiB
Groovy
plugins {
|
|
id 'java'
|
|
|
|
id 'application'
|
|
id 'jvm-test-suite'
|
|
}
|
|
|
|
java {
|
|
toolchain {
|
|
languageVersion.set(JavaLanguageVersion.of(rootProject.ext.jvmVersion))
|
|
}
|
|
}
|
|
|
|
application {
|
|
mainClass = 'nu.marginalia.converting.ConverterMain'
|
|
applicationName = 'converter-process'
|
|
}
|
|
|
|
tasks.distZip.enabled = false
|
|
|
|
apply from: "$rootProject.projectDir/srcsets.gradle"
|
|
|
|
dependencies {
|
|
implementation project(':code:common:process')
|
|
|
|
implementation project(':third-party:porterstemmer')
|
|
implementation project(':third-party:count-min-sketch')
|
|
implementation project(':third-party:parquet-floor')
|
|
|
|
implementation project(':code:index:api')
|
|
implementation project(':code:processes:process-mq-api')
|
|
|
|
implementation project(':code:common:model')
|
|
implementation project(':code:common:db')
|
|
implementation project(':code:common:service')
|
|
implementation project(':code:common:config')
|
|
implementation project(':code:libraries:message-queue')
|
|
implementation project(':code:libraries:blocking-thread-pool')
|
|
implementation project(':code:libraries:slop')
|
|
|
|
implementation project(':code:libraries:guarded-regex')
|
|
implementation project(':code:libraries:easy-lsh')
|
|
implementation project(':code:libraries:coded-sequence')
|
|
implementation project(':code:libraries:geo-ip')
|
|
implementation project(':code:libraries:language-processing')
|
|
|
|
implementation project(':code:processes:converting-process:model')
|
|
implementation project(':code:processes:crawling-process:model')
|
|
|
|
implementation project(':code:features-convert:adblock')
|
|
implementation project(':code:features-convert:anchor-keywords')
|
|
implementation project(':code:features-convert:topic-detection')
|
|
implementation project(':code:features-convert:pubdate')
|
|
implementation project(':code:features-convert:keyword-extraction')
|
|
implementation project(':code:features-convert:summary-extraction')
|
|
implementation project(':code:features-convert:stackexchange-xml')
|
|
implementation project(':code:features-convert:reddit-json')
|
|
|
|
implementation project(':code:features-crawl:crawl-blocklist')
|
|
implementation project(':code:features-crawl:link-parser')
|
|
implementation project(':code:features-crawl:content-type')
|
|
|
|
testImplementation project(':code:libraries:term-frequency-dict')
|
|
testImplementation project(':code:processes:crawling-process:model')
|
|
|
|
implementation libs.bundles.slf4j
|
|
|
|
implementation libs.notnull
|
|
implementation libs.jwarc
|
|
|
|
implementation libs.jsoup
|
|
|
|
implementation libs.guava
|
|
implementation dependencies.create(libs.guice.get()) {
|
|
exclude group: 'com.google.guava'
|
|
}
|
|
implementation libs.guava
|
|
implementation libs.bundles.gson
|
|
|
|
implementation libs.zstd
|
|
|
|
implementation libs.bundles.mariadb
|
|
implementation libs.bundles.nlp
|
|
|
|
implementation libs.roaringbitmap
|
|
implementation libs.trove
|
|
implementation libs.fastutil
|
|
|
|
implementation libs.snakeyaml
|
|
|
|
implementation libs.crawlercommons
|
|
|
|
implementation libs.commons.lang3
|
|
implementation libs.commons.compress
|
|
implementation libs.sqlite
|
|
|
|
testImplementation libs.bundles.slf4j.test
|
|
testImplementation libs.bundles.junit
|
|
testImplementation libs.mockito
|
|
|
|
testImplementation project(':code:processes:test-data')
|
|
testImplementation project(':code:processes:crawling-process')
|
|
}
|