#!/bin/bash # # This script will set up a Marginalia instance in a given directory. # It will create a docker-compose.yml file, and a directory structure # with the necessary files. It will also create a MariaDB database # in docker, and run the flyway migrations to set up the database. # # After the script is run, the instance can be started with # $ docker-compose up -d # # The instance can be stopped with # $ docker-compose down -v # # It is likely that you will want to edit the docker-compose.yml file # to change the ports that the services are exposed on, and to change # the volumes that are mounted. The default configuration is provided # a starting point. set -e if ! command -v envsubst &> /dev/null then echo "The envsubst command could not be found, please install it. It is usually part of GNU gettext." exit fi if [ -z "${1}" ]; then echo "Usage: $0 " exit 1 fi if [ -e "${1}" ]; then echo "ERROR: Destination ${1} already exists, refusing to overwrite" exit 1 fi INSTALL_DIR=$(realpath ${1}) echo "Would you like to set up a:" echo echo "1) barebones instance (1 node)" echo "2) barebones instance (2 nodes)" echo "3) full Marginalia Search instance?" echo "4) non-docker install? (not recommended)" echo read -p "Enter 1, 2, 3, or 4: " INSTANCE_TYPE ## Validate if [ "${INSTANCE_TYPE}" != "1" ] && [ "${INSTANCE_TYPE}" != "2" ] && [ "${INSTANCE_TYPE}" != "3" ] && [ "${INSTANCE_TYPE}" != "4" ]; then echo echo "ERROR: Invalid instance type, choose 1, 2 or 3" exit 1 fi echo echo "We're going to set up a Mariadb database in docker, please enter some details" read -p "MariaDB user (e.g. marginalia): " MARIADB_USER read -s -p "MariaDB password (e.g. hunter2 ;-): " MARIADB_PASSWORD echo read -s -p "MariaDB password (again): " MARIADB_PASSWORD2 echo export MARIADB_USER export MARIADB_PASSWORD if [ "${INSTANCE_TYPE}" == "4" ]; then export MARIADB_HOST="localhost" else export MARIADB_HOST="mariadb" fi if [ "${MARIADB_PASSWORD}" != "${MARIADB_PASSWORD2}" ]; then echo "ERROR: Passwords do not match" exit 1 fi echo echo "Will install to ${INSTALL_DIR}" read -p "Press enter to continue, or Ctrl-C to abort" pushd $(dirname $0) ./setup.sh ## Ensure that the setup script has been run mkdir -p ${INSTALL_DIR} echo "** Copying files to ${INSTALL_DIR}" for dir in model data conf conf/properties env; do if [ ! -d ${dir} ]; then echo "ERROR: ${dir} does not exist" exit 1 fi echo "Copying ${dir}/" mkdir -p ${INSTALL_DIR}/${dir} find ${dir} -maxdepth 1 -type f -exec cp -v {} ${INSTALL_DIR}/{} \; done # for barebones, tell the control service to hide the marginalia app specific stuff if [ "${INSTANCE_TYPE}" == "1" ]; then echo "control.hideMarginaliaApp=true" > ${INSTALL_DIR}/conf/properties/control-service.properties elif [ "${INSTANCE_TYPE}" == "2" ]; then echo "control.hideMarginaliaApp=true" > ${INSTALL_DIR}/conf/properties/control-service.properties elif [ "${INSTANCE_TYPE}" == "4" ]; then echo "control.hideMarginaliaApp=true" > ${INSTALL_DIR}/conf/properties/control-service.properties # (leading with a blank newline is important, as we cannot trust that the source file ends with a new-line) cat >>${INSTALL_DIR}/conf/properties/system.properties < ${INSTALL_DIR}/env/mariadb.env envsubst < install/db.properties.template > ${INSTALL_DIR}/conf/db.properties echo "** Creating docker-compose.yml" ## Hack to get around envstubst substituting these values, which we want to be verbatim export uval="\$\$MARIADB_USER" export pval="\$\$MARIADB_PASSWORD" export INSTALL_DIR if [ "${INSTANCE_TYPE}" == "1" ]; then envsubst < install/docker-compose-barebones-1.yml.template >${INSTALL_DIR}/docker-compose.yml elif [ "${INSTANCE_TYPE}" == "2" ]; then envsubst < install/docker-compose-barebones-2.yml.template >${INSTALL_DIR}/docker-compose.yml elif [ "${INSTANCE_TYPE}" == "3" ]; then envsubst < install/docker-compose-marginalia.yml.template >${INSTALL_DIR}/docker-compose.yml elif [ "${INSTANCE_TYPE}" == "4" ]; then envsubst < install/docker-compose-scaffold.yml.template >${INSTALL_DIR}/docker-compose.yml cat < ${INSTALL_DIR}/README Quick note about running Marginalia Search in a non-docker environment. This installation mode is not recommended, as it is significantly more complex to set up and maintain: You will need to manage the services yourself, including port management. * The template sets up a sample (in-docker) setup for mariadb and zookeeper. These can also be run outside of docker, but you will need to update the db.properties file and "zookeeper-hosts" in the system.properties file to point to the correct locations/addresses. * Each service is spawned by the same launcher. When building the project with "gradlew assemble", the launcher is put in "code/services-core/single-service-runner/build/distributions/marginalia.tar". This needs to be extracted. To launch a process you need to unpack it, and then run the launcher with the appropriate arguments. For example: WMSA_HOME=/path/to/install/dir marginalia control:1 127.0.0.1:7000:7001 127.0.0.2 This command will start the control partition 1 on ports 7000 (HTTP) and 7001 (GRPC), bound to 127.0.0.1, and it will announce its presence to the local zookeeper instance on 127.0.0.2. A working setup needs at all the services * control [ http port is the control GUI ] * query [ http port is the query GUI ] * index [ http port is internal ] * executor [ http port is internal ] Since you will need to manage ports yourself, you must assign distinct ports-pairs to each service. * An index and executor services should exist on the same partition e.g. index:1 and executor:1. The partition number is the last digit of the service name, and should be positive. You can have multiple pairs of index and executor partitions, but the pair should run on the same physical machine with the same install directory. * The query service can use any partition number. * The control service should be on partition 1. EOF echo echo "=====" cat ${INSTALL_DIR}/README echo echo "=====" echo "To read this again, look in ${INSTALL_DIR}/README" echo fi popd