Merge pull request #175000 from shyim/add-opensearch

opensearch: init at 2.5.0

nixos/doc/manual/release-notes/rl-2305.section.md

@@ -36,6 +36,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [imaginary](https://github.com/h2non/imaginary), a microservice for high-level image processing that Nextcloud can use to generate previews. Available as [services.imaginary](#opt-services.imaginary.enable).
 
+- [opensearch](https://opensearch.org), a search server alternative to Elasticsearch. Available as [services.opensearch](options.html#opt-services.opensearch.enable).
+
 - [goeland](https://github.com/slurdge/goeland), an alternative to rss2email written in golang with many filters. Available as [services.goeland](#opt-services.goeland.enable).
 
 - [atuin](https://github.com/ellie/atuin), a sync server for shell history. Available as [services.atuin](#opt-services.atuin.enable).

nixos/modules/module-list.nix

@@ -1048,6 +1048,7 @@
   ./services/search/hound.nix
   ./services/search/kibana.nix
   ./services/search/meilisearch.nix
+  ./services/search/opensearch.nix
   ./services/search/solr.nix
   ./services/security/aesmd.nix
   ./services/security/certmgr.nix

nixos/modules/services/search/opensearch.nix

@@ -0,0 +1,244 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+  cfg = config.services.opensearch;
+
+  settingsFormat = pkgs.formats.yaml {};
+
+  configDir = cfg.dataDir + "/config";
+
+  usingDefaultDataDir = cfg.dataDir == "/var/lib/opensearch";
+  usingDefaultUserAndGroup = cfg.user == "opensearch" && cfg.group == "opensearch";
+
+  opensearchYml = settingsFormat.generate "opensearch.yml" cfg.settings;
+
+  loggingConfigFilename = "log4j2.properties";
+  loggingConfigFile = pkgs.writeTextFile {
+    name = loggingConfigFilename;
+    text = cfg.logging;
+  };
+in
+{
+
+  options.services.opensearch = {
+    enable = mkEnableOption (lib.mdDoc "OpenSearch");
+
+    package = lib.mkPackageOptionMD pkgs "OpenSearch" {
+      default = [ "opensearch" ];
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = settingsFormat.type;
+
+        options."network.host" = lib.mkOption {
+          type = lib.types.str;
+          default = "127.0.0.1";
+          description = lib.mdDoc ''
+            Which port this service should listen on.
+          '';
+        };
+
+        options."cluster.name" = lib.mkOption {
+          type = lib.types.str;
+          default = "opensearch";
+          description = lib.mdDoc ''
+            The name of the cluster.
+          '';
+        };
+
+        options."discovery.type" = lib.mkOption {
+          type = lib.types.str;
+          default = "single-node";
+          description = lib.mdDoc ''
+            The type of discovery to use.
+          '';
+        };
+
+        options."http.port" = lib.mkOption {
+          type = lib.types.port;
+          default = 9200;
+          description = lib.mdDoc ''
+            The port to listen on for HTTP traffic.
+          '';
+        };
+
+        options."transport.port" = lib.mkOption {
+          type = lib.types.port;
+          default = 9300;
+          description = lib.mdDoc ''
+            The port to listen on for transport traffic.
+          '';
+        };
+      };
+
+      default = {};
+
+      description = lib.mdDoc ''
+        OpenSearch configuration.
+      '';
+    };
+
+    logging = lib.mkOption {
+      description = lib.mdDoc "opensearch logging configuration.";
+
+      default = ''
+        logger.action.name = org.opensearch.action
+        logger.action.level = info
+
+        appender.console.type = Console
+        appender.console.name = console
+        appender.console.layout.type = PatternLayout
+        appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
+
+        rootLogger.level = info
+        rootLogger.appenderRef.console.ref = console
+      '';
+      type = types.str;
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/opensearch";
+      apply = converge (removeSuffix "/");
+      description = lib.mdDoc ''
+        Data directory for OpenSearch. If you change this, you need to
+        manually create the directory. You also need to create the
+        `opensearch` user and group, or change
+        [](#opt-services.opensearch.user) and
+        [](#opt-services.opensearch.group) to existing ones with
+        access to the directory.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "opensearch";
+      description = lib.mdDoc ''
+        The user OpenSearch runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "opensearch";
+      description = lib.mdDoc ''
+        The group OpenSearch runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    extraCmdLineOptions = lib.mkOption {
+      description = lib.mdDoc "Extra command line options for the OpenSearch launcher.";
+      default = [ ];
+      type = lib.types.listOf lib.types.str;
+    };
+
+    extraJavaOptions = lib.mkOption {
+      description = lib.mdDoc "Extra command line options for Java.";
+      default = [ ];
+      type = lib.types.listOf lib.types.str;
+      example = [ "-Djava.net.preferIPv4Stack=true" ];
+    };
+
+    restartIfChanged = lib.mkOption {
+      type = lib.types.bool;
+      description = lib.mdDoc ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on a server or cluster.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = true;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.opensearch = {
+      description = "OpenSearch Daemon";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = [ pkgs.inetutils ];
+      inherit (cfg) restartIfChanged;
+      environment = {
+        OPENSEARCH_HOME = cfg.dataDir;
+        OPENSEARCH_JAVA_OPTS = toString cfg.extraJavaOptions;
+        OPENSEARCH_PATH_CONF = configDir;
+      };
+      serviceConfig = {
+        ExecStartPre =
+          let
+            startPreFullPrivileges = ''
+              set -o errexit -o pipefail -o nounset -o errtrace
+              shopt -s inherit_errexit
+            '' + (optionalString (!config.boot.isContainer) ''
+              # Only set vm.max_map_count if lower than ES required minimum
+              # This avoids conflict if configured via boot.kernel.sysctl
+              if [ $(${pkgs.procps}/bin/sysctl -n vm.max_map_count) -lt 262144 ]; then
+                ${pkgs.procps}/bin/sysctl -w vm.max_map_count=262144
+              fi
+            '');
+            startPreUnprivileged = ''
+              set -o errexit -o pipefail -o nounset -o errtrace
+              shopt -s inherit_errexit
+
+              # Install plugins
+              ln -sfT ${cfg.package}/lib ${cfg.dataDir}/lib
+              ln -sfT ${cfg.package}/modules ${cfg.dataDir}/modules
+
+              # opensearch needs to create the opensearch.keystore in the config directory
+              # so this directory needs to be writable.
+              mkdir -p ${configDir}
+              chmod 0700 ${configDir}
+
+              # Note that we copy config files from the nix store instead of symbolically linking them
+              # because otherwise X-Pack Security will raise the following exception:
+              # java.security.AccessControlException:
+              # access denied ("java.io.FilePermission" "/var/lib/opensearch/config/opensearch.yml" "read")
+
+              cp ${opensearchYml} ${configDir}/opensearch.yml
+
+              # Make sure the logging configuration for old OpenSearch versions is removed:
+              rm -f "${configDir}/logging.yml"
+              cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename}
+              mkdir -p ${configDir}/scripts
+              cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options
+
+              # redirect jvm logs to the data directory
+              mkdir -p ${cfg.dataDir}/logs
+              chmod 0700 ${cfg.dataDir}/logs
+              sed -e '#logs/gc.log#${cfg.dataDir}/logs/gc.log#' -i ${configDir}/jvm.options
+            '';
+          in [
+            "+${pkgs.writeShellScript "opensearch-start-pre-full-privileges" startPreFullPrivileges}"
+            "${pkgs.writeShellScript "opensearch-start-pre-unprivileged" startPreUnprivileged}"
+          ];
+        ExecStartPost = pkgs.writeShellScript "opensearch-start-post" ''
+          set -o errexit -o pipefail -o nounset -o errtrace
+          shopt -s inherit_errexit
+
+          # Make sure opensearch is up and running before dependents
+          # are started
+          while ! ${pkgs.curl}/bin/curl -sS -f http://${cfg.settings."network.host"}:${toString cfg.settings."http.port"} 2>/dev/null; do
+            sleep 1
+          done
+        '';
+        ExecStart = "${cfg.package}/bin/opensearch ${toString cfg.extraCmdLineOptions}";
+        User = cfg.user;
+        Group = cfg.group;
+        LimitNOFILE = "1024000";
+        Restart = "always";
+        TimeoutStartSec = "infinity";
+        DynamicUser = usingDefaultUserAndGroup && usingDefaultDataDir;
+      } // (optionalAttrs (usingDefaultDataDir) {
+        StateDirectory = "opensearch";
+        StateDirectoryMode = "0700";
+      });
+    };
+
+    environment.systemPackages = [ cfg.package ];
+  };
+}

nixos/tests/all-tests.nix

@@ -490,6 +490,7 @@ in {
   ombi = handleTest ./ombi.nix {};
   openarena = handleTest ./openarena.nix {};
   openldap = handleTest ./openldap.nix {};
+  opensearch = discoverTests (import ./opensearch.nix);
   openresty-lua = handleTest ./openresty-lua.nix {};
   opensmtpd = handleTest ./opensmtpd.nix {};
   opensmtpd-rspamd = handleTest ./opensmtpd-rspamd.nix {};

nixos/tests/opensearch.nix

@@ -0,0 +1,52 @@
+let
+  opensearchTest =
+    import ./make-test-python.nix (
+      { pkgs, lib, extraSettings ? {} }: {
+        name = "opensearch";
+        meta.maintainers = with pkgs.lib.maintainers; [ shyim ];
+
+        nodes.machine = lib.mkMerge [
+          {
+            virtualisation.memorySize = 2048;
+            services.opensearch.enable = true;
+          }
+          extraSettings
+        ];
+
+        testScript = ''
+          machine.start()
+          machine.wait_for_unit("opensearch.service")
+          machine.wait_for_open_port(9200)
+
+          machine.succeed(
+              "curl --fail localhost:9200"
+          )
+        '';
+      });
+in
+{
+  opensearch = opensearchTest {};
+  opensearchCustomPathAndUser = opensearchTest {
+    extraSettings = {
+      services.opensearch.dataDir = "/var/opensearch_test";
+      services.opensearch.user = "open_search";
+      services.opensearch.group = "open_search";
+      system.activationScripts.createDirectory = {
+        text = ''
+          mkdir -p "/var/opensearch_test"
+          chown open_search:open_search /var/opensearch_test
+          chmod 0700 /var/opensearch_test
+        '';
+        deps = [ "users" "groups" ];
+      };
+      users = {
+        groups.open_search = {};
+        users.open_search = {
+          description = "OpenSearch daemon user";
+          group = "open_search";
+          isSystemUser = true;
+        };
+      };
+    };
+  };
+}

pkgs/servers/search/opensearch/default.nix

@@ -0,0 +1,54 @@
+{ lib
+, stdenvNoCC
+, fetchurl
+, makeWrapper
+, jre_headless
+, util-linux
+, gnugrep
+, coreutils
+, autoPatchelfHook
+, zlib
+, nixosTests
+}:
+
+stdenvNoCC.mkDerivation rec {
+  pname = "opensearch";
+  version = "2.5.0";
+
+  src = fetchurl {
+    url = "https://artifacts.opensearch.org/releases/bundle/opensearch/${version}/opensearch-${version}-linux-x64.tar.gz";
+    hash = "sha256-WPD5StVBb/hK+kP/1wkQQBKRQma/uaP+8ULeIFUBL1U=";
+  };
+
+  nativeBuildInputs = [ makeWrapper ];
+  buildInputs = [ jre_headless util-linux ];
+  patches = [./opensearch-home-fix.patch ];
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out
+    cp -R bin config lib modules plugins $out
+
+    substituteInPlace $out/bin/opensearch \
+      --replace 'bin/opensearch-keystore' "$out/bin/opensearch-keystore"
+
+    wrapProgram $out/bin/opensearch \
+      --prefix PATH : "${lib.makeBinPath [ util-linux gnugrep coreutils ]}" \
+      --set JAVA_HOME "${jre_headless}"
+
+    wrapProgram $out/bin/opensearch-plugin --set JAVA_HOME "${jre_headless}"
+
+    runHook postInstall
+  '';
+
+  passthru.tests = nixosTests.opensearch;
+
+  meta = {
+    description = "Open Source, Distributed, RESTful Search Engine";
+    homepage = "https://github.com/opensearch-project/OpenSearch";
+    license = lib.licenses.asl20;
+    platforms = lib.platforms.unix;
+    maintainers = with lib.maintainers; [ shyim ];
+  };
+}

pkgs/servers/search/opensearch/opensearch-home-fix.patch

@@ -0,0 +1,26 @@
+diff -Naur a/bin/opensearch-env b/bin/opensearch-env
+--- a/bin/opensearch-env	2017-12-12 13:31:51.000000000 +0100
++++ b/bin/opensearch-env	2017-12-18 19:51:12.282809695 +0100
+@@ -19,18 +19,10 @@
+   fi
+ done
+ 
+-# determine OpenSearch home; to do this, we strip from the path until we find
+-# bin, and then strip bin (there is an assumption here that there is no nested
+-# directory under bin also named bin)
+-OPENSEARCH_HOME=`dirname "$SCRIPT"`
+-
+-# now make OPENSEARCH_HOME absolute
+-OPENSEARCH_HOME=`cd "$OPENSEARCH_HOME"; pwd`
+-
+-while [ "`basename "$OPENSEARCH_HOME"`" != "bin" ]; do
+-  OPENSEARCH_HOME=`dirname "$OPENSEARCH_HOME"`
+-done
+-OPENSEARCH_HOME=`dirname "$OPENSEARCH_HOME"`
++if [ -z "$OPENSEARCH_HOME" ]; then
++    echo "You must set the OPENSEARCH_HOME var" >&2
++    exit 1
++fi
+ 
+ # now set the classpath
+ OPENSEARCH_CLASSPATH="$OPENSEARCH_HOME/lib/*"

pkgs/top-level/all-packages.nix

@@ -34819,6 +34819,8 @@ with pkgs;
 
   openrct2 = callPackage ../games/openrct2 { };
 
+  opensearch = callPackage ../servers/search/opensearch { };
+
   osu-lazer = callPackage ../games/osu-lazer { };
 
   osu-lazer-bin = callPackage ../games/osu-lazer/bin.nix { };