elasticsearch: add 5.x package, service

nixos/modules/services/search/elasticsearch.nix

@@ -5,13 +5,22 @@ with lib;
 let
   cfg = config.services.elasticsearch;
 
+  es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0;
+
   esConfig = ''
     network.host: ${cfg.listenAddress}
-    network.port: ${toString cfg.port}
-    network.tcp.port: ${toString cfg.tcp_port}
-    # TODO: find a way to enable security manager
-    security.manager.enabled: false
     cluster.name: ${cfg.cluster_name}
+
+    ${if es5 then ''
+      http.port: ${toString cfg.port}
+      transport.tcp.port: ${toString cfg.tcp_port}
+    '' else ''
+      network.port: ${toString cfg.port}
+      network.tcp.port: ${toString cfg.tcp_port}
+      # TODO: find a way to enable security manager
+      security.manager.enabled: false
+    ''}
+
     ${cfg.extraConf}
   '';
 
@@ -19,13 +28,18 @@ let
     name = "elasticsearch-config";
     paths = [
       (pkgs.writeTextDir "elasticsearch.yml" esConfig)
-      (pkgs.writeTextDir "logging.yml" cfg.logging)
+      (if es5 then (pkgs.writeTextDir "log4j2.properties" cfg.logging)
+              else (pkgs.writeTextDir "logging.yml" cfg.logging))
     ];
+    # Elasticsearch 5.x won't start when the scripts directory does not exist
+    postBuild = if es5 then "${pkgs.coreutils}/bin/mkdir -p $out/scripts" else "";
   };
 
   esPlugins = pkgs.buildEnv {
     name = "elasticsearch-plugins";
     paths = cfg.plugins;
+    # Elasticsearch 5.x won't start when the plugins directory does not exist
+    postBuild = if es5 then "${pkgs.coreutils}/bin/mkdir -p $out/plugins" else "";
   };
 
 in {
@@ -85,18 +99,30 @@ in {
 
     logging = mkOption {
       description = "Elasticsearch logging configuration.";
-      default = ''
+      default =
-        rootLogger: INFO, console
+        if es5 then ''
-        logger:
+          logger.action.name = org.elasticsearch.action
-          action: INFO
+          logger.action.level = info
-          com.amazonaws: WARN
+
-        appender:
+          appender.console.type = Console
-          console:
+          appender.console.name = console
-            type: console
+          appender.console.layout.type = PatternLayout
-            layout:
+          appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-              type: consolePattern
+
-              conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+          rootLogger.level = info
-      '';
+          rootLogger.appenderRef.console.ref = console
+        '' else ''
+          rootLogger: INFO, console
+          logger:
+            action: INFO
+            com.amazonaws: WARN
+          appender:
+            console:
+              type: console
+              layout:
+                type: consolePattern
+                conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
+        '';
       type = types.str;
     };
 
@@ -112,6 +138,12 @@ in {
       description = "Extra command line options for the elasticsearch launcher.";
       default = [];
       type = types.listOf types.str;
+    };
+
+    extraJavaOptions = mkOption {
+      description = "Extra command line options for Java.";
+      default = [];
+      type = types.listOf types.str;
       example = [ "-Djava.net.preferIPv4Stack=true" ];
     };
 
@@ -133,13 +165,21 @@ in {
       path = [ pkgs.inetutils ];
       environment = {
         ES_HOME = cfg.dataDir;
+        ES_JAVA_OPTS = toString ([ "-Des.path.conf=${configDir}" ] ++ cfg.extraJavaOptions);
       };
       serviceConfig = {
-        ExecStart = "${cfg.package}/bin/elasticsearch -Des.path.conf=${configDir} ${toString cfg.extraCmdLineOptions}";
+        ExecStart = "${cfg.package}/bin/elasticsearch ${toString cfg.extraCmdLineOptions}";
         User = "elasticsearch";
         PermissionsStartOnly = true;
+        LimitNOFILE = "1024000";
       };
       preStart = ''
+        # Only set vm.max_map_count if lower than ES required minimum
+        # This avoids conflict if configured via boot.kernel.sysctl
+        if [ `${pkgs.procps}/bin/sysctl -n vm.max_map_count` -lt 262144 ]; then
+          ${pkgs.procps}/bin/sysctl -w vm.max_map_count=262144
+        fi
+
         mkdir -m 0700 -p ${cfg.dataDir}
 
         # Install plugins

pkgs/servers/search/elasticsearch/5.x.nix

@@ -0,0 +1,44 @@
+{ stdenv, fetchurl, makeWrapper, jre, utillinux, getopt }:
+
+with stdenv.lib;
+
+stdenv.mkDerivation rec {
+  version = "5.4.0";
+  name = "elasticsearch-${version}";
+
+  src = fetchurl {
+    url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
+    sha256 = "1ml2dvwxxhj3azj13wa8xd08kpapal2477lpcaxzw5gnzizgyx5z";
+  };
+
+  patches = [ ./es-home-5.x.patch ./es-classpath-5.x.patch ];
+
+  buildInputs = [ makeWrapper jre ] ++
+    (if (!stdenv.isDarwin) then [utillinux] else [getopt]);
+
+  installPhase = ''
+    mkdir -p $out
+    cp -R bin config lib modules plugins $out
+
+    chmod -x $out/bin/*.*
+
+    wrapProgram $out/bin/elasticsearch \
+      --prefix ES_CLASSPATH : "$out/lib/*" \
+      ${if (!stdenv.isDarwin)
+        then ''--prefix PATH : "${utillinux}/bin/"''
+        else ''--prefix PATH : "${getopt}/bin"''} \
+      --set JAVA_HOME "${jre}" \
+      --set ES_JVM_OPTIONS "$out/config/jvm.options"
+
+    wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre}"
+  '';
+
+  meta = {
+    description = "Open Source, Distributed, RESTful Search Engine";
+    license = licenses.asl20;
+    platforms = platforms.unix;
+    maintainers = [
+      maintainers.apeschar
+    ];
+  };
+}

pkgs/servers/search/elasticsearch/es-classpath-5.x.patch

@@ -0,0 +1,34 @@
+diff -rupN a/bin/elasticsearch b/bin/elasticsearch
+--- a/bin/elasticsearch	2017-05-17 10:53:49.444487071 +0200
++++ b/bin/elasticsearch	2017-05-17 10:55:52.755081523 +0200
+@@ -129,12 +129,7 @@ ES_JAVA_OPTS="$(parse_jvm_options "$ES_J
+ # If an include wasn't specified in the environment, then search for one...
+ if [ "x$ES_INCLUDE" = "x" ]; then
+     # Locations (in order) to use when searching for an include file.
+-    for include in /usr/share/elasticsearch/elasticsearch.in.sh \
+-                   /usr/local/share/elasticsearch/elasticsearch.in.sh \
+-                   /opt/elasticsearch/elasticsearch.in.sh \
+-                   ~/.elasticsearch.in.sh \
+-                   "$ES_HOME/bin/elasticsearch.in.sh" \
+-                   "`dirname "$0"`"/elasticsearch.in.sh; do
++    for include in "`dirname "$0"`"/elasticsearch.in.sh; do
+         if [ -r "$include" ]; then
+             . "$include"
+             break
+diff -rupN a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh
+--- a/bin/elasticsearch.in.sh	2017-04-28 19:41:47.000000000 +0200
++++ b/bin/elasticsearch.in.sh	2017-05-17 10:56:49.303519788 +0200
+@@ -1,13 +1 @@
+ #!/bin/bash
+-
+-# check in case a user was using this mechanism
+-if [ "x$ES_CLASSPATH" != "x" ]; then
+-    cat >&2 << EOF
+-Error: Don't modify the classpath with ES_CLASSPATH. Best is to add
+-additional elements via the plugin mechanism, or if code must really be
+-added to the main classpath, add jars to lib/ (unsupported).
+-EOF
+-    exit 1
+-fi
+-
+-ES_CLASSPATH="$ES_HOME/lib/*"

pkgs/servers/search/elasticsearch/es-home-5.x.patch

@@ -0,0 +1,31 @@
+diff -rupN a/bin/elasticsearch b/bin/elasticsearch
+--- a/bin/elasticsearch	2017-05-17 10:53:42.214686741 +0200
++++ b/bin/elasticsearch	2017-05-17 10:53:49.444487071 +0200
+@@ -105,7 +105,11 @@ while [ -h "$SCRIPT" ] ; do
+ done
+ 
+ # determine elasticsearch home
+-ES_HOME=`dirname "$SCRIPT"`/..
++
++if [ -z "$ES_HOME" ]; then
++    echo "You must set the ES_HOME var" >&2
++    exit 1
++fi
+ 
+ # make ELASTICSEARCH_HOME absolute
+ ES_HOME=`cd "$ES_HOME"; pwd`
+diff -rupN a/bin/elasticsearch-plugin b/bin/elasticsearch-plugin
+--- a/bin/elasticsearch-plugin	2017-05-17 10:53:42.214686741 +0200
++++ b/bin/elasticsearch-plugin	2017-05-17 10:53:49.445487044 +0200
+@@ -16,7 +16,10 @@ while [ -h "$SCRIPT" ] ; do
+ done
+ 
+ # determine elasticsearch home
+-ES_HOME=`dirname "$SCRIPT"`/..
++if [ -z "$ES_HOME" ]; then
++    echo "You must set the ES_HOME var" >&2
++    exit 1
++fi
+ 
+ # make ELASTICSEARCH_HOME absolute
+ ES_HOME=`cd "$ES_HOME"; pwd`

pkgs/top-level/all-packages.nix

@@ -1731,6 +1731,7 @@ with pkgs;
 
   elasticsearch = callPackage ../servers/search/elasticsearch { };
   elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
+  elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
 
   elasticsearchPlugins = recurseIntoAttrs (
     callPackage ../servers/search/elasticsearch/plugins.nix { }